gitlab-qa 6.14.0 → 6.15.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +22 -26
- data/.gitlab/merge_request_templates/Release.md +1 -1
- data/docs/release_process.md +7 -6
- data/lib/gitlab/qa/report/generate_test_session.rb +18 -7
- data/lib/gitlab/qa/report/prepare_stage_reports.rb +4 -1
- data/lib/gitlab/qa/report/relate_failure_issue.rb +4 -3
- data/lib/gitlab/qa/report/report_as_issue.rb +5 -7
- data/lib/gitlab/qa/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 40d1c050d428a575441e8608a95d08de4e683fcce3e761407b3af5d528ed5fb0
|
4
|
+
data.tar.gz: f94798ad1e5e0f7dec06421bca8dabd7bf6cb9fef05afe8842072bb1734507b7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f040e24b9bea5771f89f18088b31d99910827922dec8e9cea7b2e8e59a932aa6b7ecc7e93024c4af026cbe4da287af83703fb758dca7395899e833e8c8018410
|
7
|
+
data.tar.gz: 13213f41ff01e4429b66c64cbe25347bab733785f76417c94696732cd3f0c6502ea8a845c2e43af7c452225bdcf6ea7c66ae7df739c2d1d3d2253d3da838dbab
|
data/.gitlab-ci.yml
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
stages:
|
2
2
|
- check
|
3
|
-
- release
|
4
3
|
- test
|
5
4
|
- report
|
5
|
+
- deploy
|
6
6
|
- notify
|
7
7
|
|
8
8
|
default:
|
@@ -35,13 +35,6 @@ workflow:
|
|
35
35
|
# For triggers from GitLab MR pipelines (and pipelines from other projects), create a pipeline
|
36
36
|
- if: '$CI_PIPELINE_SOURCE == "pipeline"'
|
37
37
|
|
38
|
-
.default-rules:
|
39
|
-
rules:
|
40
|
-
- if: '$CI_COMMIT_TAG || $RELEASE'
|
41
|
-
when: never
|
42
|
-
- if: '$RELEASE == null && $CI_JOB_NAME =~ /staging/'
|
43
|
-
when: manual
|
44
|
-
|
45
38
|
variables:
|
46
39
|
DOCKER_DRIVER: overlay2
|
47
40
|
DOCKER_HOST: tcp://docker:2375
|
@@ -65,21 +58,6 @@ rubocop:
|
|
65
58
|
rspec:
|
66
59
|
extends: .check-base
|
67
60
|
|
68
|
-
release:
|
69
|
-
stage: release
|
70
|
-
rules:
|
71
|
-
- if: '$CI_COMMIT_TAG'
|
72
|
-
script:
|
73
|
-
- gem update --system
|
74
|
-
- ruby --version
|
75
|
-
- gem env version
|
76
|
-
- gem build gitlab-qa.gemspec
|
77
|
-
- gem push gitlab-qa*.gem
|
78
|
-
artifacts:
|
79
|
-
paths:
|
80
|
-
- gitlab-qa*.gem
|
81
|
-
expire_in: 30 days
|
82
|
-
|
83
61
|
.test:
|
84
62
|
stage: test
|
85
63
|
services:
|
@@ -107,6 +85,10 @@ release:
|
|
107
85
|
variables:
|
108
86
|
DEFAULT_RELEASE: "CE"
|
109
87
|
rules:
|
88
|
+
# Don't run E2E jobs on release pipelines
|
89
|
+
- if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab-qa" && $RELEASE == null'
|
90
|
+
changes: ["lib/**/version.rb"]
|
91
|
+
when: never
|
110
92
|
- if: '$CI_COMMIT_TAG || $RELEASE =~ /gitlab-ee/'
|
111
93
|
when: never
|
112
94
|
- if: '$RELEASE == null && $CI_JOB_NAME =~ /quarantine|custom/'
|
@@ -121,6 +103,10 @@ release:
|
|
121
103
|
variables:
|
122
104
|
DEFAULT_RELEASE: "EE"
|
123
105
|
rules:
|
106
|
+
# Don't run E2E jobs on release pipelines
|
107
|
+
- if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab-qa" && $RELEASE == null'
|
108
|
+
changes: ["lib/**/version.rb"]
|
109
|
+
when: never
|
124
110
|
- if: '$CI_COMMIT_TAG || $RELEASE =~ /gitlab-ce/'
|
125
111
|
when: never
|
126
112
|
- if: '$RELEASE == null && $CI_JOB_NAME =~ /quarantine|custom/'
|
@@ -132,7 +118,15 @@ release:
|
|
132
118
|
- if: '$RELEASE == null || $RELEASE =~ /gitlab-ee/ || $CI_MERGE_REQUEST_ID || $CI_COMMIT_REF_NAME == "master"'
|
133
119
|
|
134
120
|
.only-qa:
|
135
|
-
|
121
|
+
rules:
|
122
|
+
# Don't run E2E jobs on release pipelines
|
123
|
+
- if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab-qa" && $RELEASE == null'
|
124
|
+
changes: ["lib/**/version.rb"]
|
125
|
+
when: never
|
126
|
+
- if: '$CI_COMMIT_TAG || $RELEASE'
|
127
|
+
when: never
|
128
|
+
- if: '$RELEASE == null && $CI_JOB_NAME =~ /staging/'
|
129
|
+
when: manual
|
136
130
|
|
137
131
|
.high-capacity:
|
138
132
|
tags:
|
@@ -474,7 +468,6 @@ ee:mattermost-quarantine:
|
|
474
468
|
ee:geo:
|
475
469
|
extends:
|
476
470
|
- .test
|
477
|
-
- .high-capacity
|
478
471
|
- .ee-qa
|
479
472
|
- .rspec-report-opts
|
480
473
|
variables:
|
@@ -483,7 +476,6 @@ ee:geo:
|
|
483
476
|
ee:geo-quarantine:
|
484
477
|
extends:
|
485
478
|
- .test
|
486
|
-
- .high-capacity
|
487
479
|
- .ee-qa
|
488
480
|
- .quarantine
|
489
481
|
- .rspec-report-opts
|
@@ -1012,3 +1004,7 @@ notify_slack:
|
|
1012
1004
|
- echo "CI_PIPELINE_URL is $CI_PIPELINE_URL"
|
1013
1005
|
- echo "TOP_UPSTREAM_SOURCE_JOB is $TOP_UPSTREAM_SOURCE_JOB"
|
1014
1006
|
- 'bin/slack $NOTIFY_CHANNEL "☠️ QA against $RELEASE failed! ☠️ See the test session report: $(cat REPORT_ISSUE_URL), and pipeline: $CI_PIPELINE_URL (triggered from $TOP_UPSTREAM_SOURCE_JOB)" ci_failing'
|
1007
|
+
|
1008
|
+
include:
|
1009
|
+
- project: 'gitlab-org/quality/pipeline-common'
|
1010
|
+
file: '/ci/gem-release.yml'
|
@@ -30,6 +30,6 @@ with the latest commit from https://gitlab.com/gitlab-org/gitlab-qa/commits/mast
|
|
30
30
|
- [ ] Release notes are accurate.
|
31
31
|
|
32
32
|
- Checklist after merging:
|
33
|
-
- [ ] [
|
33
|
+
- [ ] [Update the release notes for the newly created tag](docs/release_process.md#how-to).
|
34
34
|
|
35
35
|
/label ~Quality ~"feature::maintenance"
|
data/docs/release_process.md
CHANGED
@@ -24,12 +24,13 @@ when we make a change - no matter the size of the change.
|
|
24
24
|
- If not, update [`lib/gitlab/qa/version.rb`] to an appropriate [semantic version](https://semver.org) in a new merge request using the [release template](https://gitlab.com/gitlab-org/gitlab-qa/blob/master/.gitlab/merge_request_templates/Release.md)
|
25
25
|
and title the MR like `"Bump version to <version>"`.
|
26
26
|
- Merge the merge request.
|
27
|
-
-
|
28
|
-
|
29
|
-
* **Message**: Set it to "Version x.y.z", e.g. if the version is `4.7.1`, the message would be "Version 4.7.1".
|
27
|
+
- The new version should automatically be tagged and pushed to Rubygems by the `gem-release` job in the merge commit pipeline.
|
28
|
+
- Update the release notes for the newly created tag (https://gitlab.com/gitlab-org/gitlab-qa/-/tags):
|
30
29
|
* **Release notes**: Copy the release notes from the merge request.
|
31
|
-
|
32
|
-
|
33
|
-
|
30
|
+
|
31
|
+
Note: The `gem-release` job uses:
|
32
|
+
|
33
|
+
- the `GITLAB_API_TOKEN` environment variable to authenticate against GitLab.com's API in order to create the tag
|
34
|
+
- the `GEM_HOST_API_KEY` environment variable to authenticate against Rubygems.org's API in order to release the gem
|
34
35
|
|
35
36
|
[`lib/gitlab/qa/version.rb`]: https://gitlab.com/gitlab-org/gitlab-qa/blob/master/lib/gitlab/qa/version.rb#L3
|
@@ -17,9 +17,9 @@ module Gitlab
|
|
17
17
|
end
|
18
18
|
|
19
19
|
issue = gitlab.create_issue(
|
20
|
-
title: "Test session report | #{
|
20
|
+
title: "Test session report | #{pipeline}",
|
21
21
|
description: generate_description(tests),
|
22
|
-
labels: ['Quality', 'QA', 'triage report',
|
22
|
+
labels: ['Quality', 'QA', 'triage report', pipeline_name_label]
|
23
23
|
)
|
24
24
|
|
25
25
|
File.write('REPORT_ISSUE_URL', issue.web_url)
|
@@ -27,20 +27,19 @@ module Gitlab
|
|
27
27
|
# rubocop:enable Metrics/AbcSize
|
28
28
|
|
29
29
|
def generate_description(tests)
|
30
|
-
<<~MARKDOWN
|
30
|
+
<<~MARKDOWN.rstrip
|
31
31
|
## Session summary
|
32
32
|
|
33
33
|
* Deploy version: #{Runtime::Env.deploy_version}
|
34
|
-
*
|
34
|
+
* Deploy environment: #{Runtime::Env.deploy_environment}
|
35
|
+
* Pipeline: #{Runtime::Env.pipeline_from_project_name} [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
|
35
36
|
#{generate_summary(tests: tests)}
|
36
37
|
|
37
38
|
#{generate_failed_jobs_listing}
|
38
39
|
|
39
40
|
#{generate_stages_listing(tests)}
|
40
41
|
|
41
|
-
|
42
|
-
|
43
|
-
* #{Runtime::Env.qa_issue_url}
|
42
|
+
#{generate_qa_issue_relation}
|
44
43
|
MARKDOWN
|
45
44
|
end
|
46
45
|
|
@@ -228,6 +227,18 @@ module Gitlab
|
|
228
227
|
'-'
|
229
228
|
end
|
230
229
|
end
|
230
|
+
|
231
|
+
def generate_qa_issue_relation
|
232
|
+
return unless Runtime::Env.qa_issue_url
|
233
|
+
|
234
|
+
<<~MARKDOWN.chomp
|
235
|
+
## Release QA issue
|
236
|
+
|
237
|
+
* #{Runtime::Env.qa_issue_url}
|
238
|
+
|
239
|
+
/relate #{Runtime::Env.qa_issue_url}
|
240
|
+
MARKDOWN
|
241
|
+
end
|
231
242
|
end
|
232
243
|
end
|
233
244
|
end
|
@@ -34,7 +34,10 @@ module Gitlab
|
|
34
34
|
# /qa/specs/features/ee/api/<stage>
|
35
35
|
# /qa/specs/features/ee/browser_ui/<stage>
|
36
36
|
# For now we assume the Stage is whatever follows api/ or browser_ui/
|
37
|
-
|
37
|
+
test_path_match = test['file'].match(%r{(api|browser_ui)/([a-z0-9_]+)}i)
|
38
|
+
next unless test_path_match
|
39
|
+
|
40
|
+
stage = strip_number_prefix(test_path_match[2])
|
38
41
|
testcases[stage] = [] unless testcases.key?(stage)
|
39
42
|
testcases[stage] << test
|
40
43
|
end
|
@@ -10,7 +10,7 @@ module Gitlab
|
|
10
10
|
# Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
|
11
11
|
class RelateFailureIssue < ReportAsIssue
|
12
12
|
DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.05
|
13
|
-
STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s
|
13
|
+
STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s*(?!.*Failure/Error:)?(.+)(\1)}m.freeze
|
14
14
|
NEW_ISSUE_LABELS = Set.new(%w[QA Quality test failure::investigating priority::2]).freeze
|
15
15
|
|
16
16
|
MultipleIssuesFound = Class.new(StandardError)
|
@@ -85,7 +85,8 @@ module Gitlab
|
|
85
85
|
next unless relevant_issue_stacktrace
|
86
86
|
|
87
87
|
distance = ld.call(first_test_failure_stacktrace, relevant_issue_stacktrace)
|
88
|
-
diff_ratio = (distance.to_f / first_test_failure_stacktrace.size).round(3)
|
88
|
+
diff_ratio = distance.zero? ? 0.0 : (distance.to_f / first_test_failure_stacktrace.size).round(3)
|
89
|
+
|
89
90
|
if diff_ratio <= max_diff_ratio
|
90
91
|
puts " => [DEBUG] Issue #{issue} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
|
91
92
|
# The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
|
@@ -142,7 +143,7 @@ module Gitlab
|
|
142
143
|
end
|
143
144
|
|
144
145
|
def up_to_date_labels(test:, issue: nil)
|
145
|
-
super <<
|
146
|
+
super << pipeline_name_label
|
146
147
|
end
|
147
148
|
|
148
149
|
def post_failed_job_note(issue, test)
|
@@ -104,20 +104,18 @@ module Gitlab
|
|
104
104
|
labels
|
105
105
|
end
|
106
106
|
|
107
|
-
def
|
108
|
-
|
109
|
-
|
110
|
-
case environment
|
107
|
+
def pipeline_name_label
|
108
|
+
case pipeline
|
111
109
|
when 'production'
|
112
110
|
'found:gitlab.com'
|
113
111
|
when 'canary', 'staging'
|
114
|
-
"found:#{
|
112
|
+
"found:#{pipeline}.gitlab.com"
|
115
113
|
when 'preprod'
|
116
114
|
'found:pre.gitlab.com'
|
117
115
|
when 'staging-orchestrated', 'nightly', 'master'
|
118
|
-
"found:#{
|
116
|
+
"found:#{pipeline}"
|
119
117
|
else
|
120
|
-
raise "No `found:*` label for the `#{
|
118
|
+
raise "No `found:*` label for the `#{pipeline}` pipeline!"
|
121
119
|
end
|
122
120
|
end
|
123
121
|
|
data/lib/gitlab/qa/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-qa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 6.
|
4
|
+
version: 6.15.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Grzegorz Bizon
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-12-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|