gitlab_quality-test_tooling 0.2.0 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +150 -9
- data/exe/generate-test-session +2 -1
- data/exe/post-to-slack +2 -1
- data/exe/prepare-stage-reports +2 -2
- data/exe/relate-failure-issue +3 -3
- data/exe/report-results +2 -2
- data/exe/update-screenshot-paths +3 -3
- data/lib/gitlab_quality/test_tooling/gitlab_issue_client.rb +2 -2
- data/lib/gitlab_quality/test_tooling/report/prepare_stage_reports.rb +0 -1
- data/lib/gitlab_quality/test_tooling/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b6aca93da1e251be67dab8c377c542ddfdba84392efb41949b62b42fabb283d3
|
4
|
+
data.tar.gz: 32d0c338138afaeaf0f47a38cced840387ed07c55e63631847bc59a3bfb20842
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3af4b645bb8c495ed38ce06964de0008e31fbd2a097f374339874aa9f7d9b1967cd6b7c3fae7ad46188120ece0103058864e9601a68e38812002529df6ac89c0
|
7
|
+
data.tar.gz: 6c1670e30153abb2af66e098c6a89fc13140d71bf1b7aa82ed06793006c8c702bfd5d4345ced0737168f7efb6c288734828a1726a905e16d4d97f2785bc9692b
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -1,28 +1,169 @@
|
|
1
1
|
# `GitlabQuality::TestTooling`
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/gitlab_quality/test_tooling`. To experiment with that code, run `bin/console` for an interactive prompt.
|
3
|
+
This gem provides test tooling that can be used by different projects or different part of the same project, mostly in CI scripts.
|
6
4
|
|
7
5
|
## Installation
|
8
6
|
|
9
|
-
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'gitlab_quality-test_tooling', require: false
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
10
14
|
|
11
|
-
|
15
|
+
```sh
|
16
|
+
$ bundle install
|
17
|
+
```
|
12
18
|
|
13
|
-
|
19
|
+
Or install it yourself as:
|
14
20
|
|
15
|
-
|
21
|
+
```sh
|
22
|
+
$ gem install gitlab_quality-test_tooling
|
23
|
+
```
|
16
24
|
|
17
25
|
## Usage
|
18
26
|
|
19
|
-
|
27
|
+
The gem provides the following executables.
|
28
|
+
|
29
|
+
### `generate-test-session`
|
30
|
+
|
31
|
+
```shell
|
32
|
+
Purpose: Generate test session report based on RSpec report files (JSON or JUnit XML)
|
33
|
+
Usage: exe/generate-test-session [options]
|
34
|
+
-i, --input-files FILES RSpec report files (JSON or JUnit XML)
|
35
|
+
-p, --project PROJECT Can be an integer or a group/project string
|
36
|
+
-t, --token ACCESS_TOKEN A valid access token with Reporter permission in PROJECT
|
37
|
+
--dry-run Perform a dry-run (don't create or update issues or test cases)
|
38
|
+
-v, --version Show the version
|
39
|
+
-h, --help Show the usage
|
40
|
+
```
|
41
|
+
|
42
|
+
### `post-to-slack`
|
43
|
+
|
44
|
+
```shell
|
45
|
+
$ exe/post-to-slack -h
|
46
|
+
Purpose: Post a message to Slack, and optionally add a test summary table based on RSpec report files (JUnit XML)
|
47
|
+
Usage: exe/post-to-slack [options]
|
48
|
+
-w SLACK_WEBHOOK_URL, Slack webhook URL
|
49
|
+
--slack-webhook-url
|
50
|
+
-c, --channel CHANNEL Slack channel to post the message to
|
51
|
+
-m, --message MESSAGE Post message to Slack
|
52
|
+
-t FILES, Add a test summary table based on RSpec report files (JUnit XML)
|
53
|
+
--include-summary-table
|
54
|
+
-u, --username USERNAME Username to use for the Slack message
|
55
|
+
-i, --icon-emoji ICON_EMOJI Icon emoji to use for the Slack message
|
56
|
+
-v, --version Show the version
|
57
|
+
-h, --help Show the usage
|
58
|
+
```
|
59
|
+
|
60
|
+
### `prepare-stage-reports`
|
61
|
+
|
62
|
+
```shell
|
63
|
+
$ exe/prepare-stage-reports -h
|
64
|
+
Purpose: Prepare separate reports for each DevOps stage from the provided RSpec report files (JUnit XML)
|
65
|
+
Usage: exe/prepare-stage-reports [options]
|
66
|
+
-j, --junit-files FILES RSpec report files (JUnit XML)
|
67
|
+
-v, --version Show the version
|
68
|
+
-h, --help Show the usage
|
69
|
+
```
|
70
|
+
|
71
|
+
### `relate-failure-issue`
|
72
|
+
|
73
|
+
```shell
|
74
|
+
$ exe/relate-failure-issue -h
|
75
|
+
Purpose: Relate test failures to failure issues from RSpec report files (JSON or JUnit XML)
|
76
|
+
Usage: exe/relate-failure-issue [options]
|
77
|
+
-i, --input-files FILES RSpec report files (JSON or JUnit XML)
|
78
|
+
--max-diff-ratio DIFF_RATO Max stacktrace diff ratio for QA failure issues detection
|
79
|
+
-p, --project PROJECT Can be an integer or a group/project string
|
80
|
+
-t, --token ACCESS_TOKEN A valid access token with Reporter permission in PROJECT
|
81
|
+
--system-log-files SYSTEM_LOG_FILES
|
82
|
+
Include errors from system logs in failure issues
|
83
|
+
--dry-run Perform a dry-run (don't create or update issues)
|
84
|
+
-v, --version Show the version
|
85
|
+
-h, --help Show the usage
|
86
|
+
```
|
87
|
+
|
88
|
+
### `report-results`
|
89
|
+
|
90
|
+
```shell
|
91
|
+
$ exe/report-results -h
|
92
|
+
Purpose: Report test results from RSpec report files (JSON or JUnit XML) in GitLab test cases and result issues
|
93
|
+
Usage: exe/report-results [options]
|
94
|
+
-i, --input-files FILES RSpec report files (JSON or JUnit XML)
|
95
|
+
--test-case-project TEST_CASE_PROJECT
|
96
|
+
Can be an integer or a group/project string
|
97
|
+
--results-issue-project RESULTS_ISSUE_PROJECT
|
98
|
+
Can be an integer or a group/project string
|
99
|
+
-t, --token ACCESS_TOKEN A valid access token
|
100
|
+
--dry-run Perform a dry-run (don't create or update issues or test cases)
|
101
|
+
-v, --version Show the version
|
102
|
+
-h, --help Show the usage
|
103
|
+
```
|
104
|
+
|
105
|
+
### `update-screenshot-paths`
|
106
|
+
|
107
|
+
```shell
|
108
|
+
$ exe/update-screenshot-paths -h
|
109
|
+
Purpose: Update the path to screenshots to container's host from RSpec report files (JSON or JUnit XML)
|
110
|
+
Usage: exe/update-screenshot-paths [options]
|
111
|
+
-i, --input-files FILES RSpec report files (JSON or JUnit XML)
|
112
|
+
-v, --version Show the version
|
113
|
+
-h, --help Show the usage
|
114
|
+
```
|
20
115
|
|
21
116
|
## Development
|
22
117
|
|
118
|
+
### Initial setup
|
119
|
+
|
23
120
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
24
121
|
|
25
|
-
To install this gem onto your local machine, run `bundle exec rake install`.
|
122
|
+
To install this gem onto your local machine, run `bundle exec rake install`.
|
123
|
+
|
124
|
+
### Activate lefthook locally
|
125
|
+
|
126
|
+
```shell
|
127
|
+
lefthook install
|
128
|
+
```
|
129
|
+
|
130
|
+
### Testing unreleased changes in merge requests
|
131
|
+
|
132
|
+
To test an unreleased change in an actual merge request, you can create a merge request that will install the unreleased version of `gitlab_quality-test_tooling`. Bundler can install gems by specifying a repository and a revision from Git.
|
133
|
+
|
134
|
+
For example, to test `gitlab_quality-test_tooling` changes from the `your-branch-name` branch in [`gitlab-org/gitlab`](https://gitlab.com/gitlab-org/gitlab), in the `Gemfile`:
|
135
|
+
|
136
|
+
```ruby
|
137
|
+
group :development, :test, :danger do
|
138
|
+
gem 'gitlab_quality-test_tooling', '~> 3.4.3', require: false,
|
139
|
+
git: 'https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling.git',
|
140
|
+
ref: 'your-branch-name'
|
141
|
+
end
|
142
|
+
```
|
143
|
+
|
144
|
+
See an [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/92580) for more details.
|
145
|
+
|
146
|
+
## Release
|
147
|
+
|
148
|
+
[Automated gem release process](https://gitlab.com/gitlab-org/quality/pipeline-common#release-process) is used to release new version of `gitlab_quality-test_tooling` through pipelines, and this will:
|
149
|
+
|
150
|
+
- Publish the gem: https://rubygems.org/gems/gitlab_quality-test_tooling
|
151
|
+
- Add a release in the `gitlab_quality-test_tooling` project: https://gitlab.com/gitlab-org/ruby/gems/gitlab_quality-test_tooling/-/releases
|
152
|
+
- Populate the release log with the API contents. For example: https://gitlab.com/api/v4/projects/19861191/repository/changelog?version=3.4.4
|
153
|
+
|
154
|
+
### Before release
|
155
|
+
|
156
|
+
Make sure to include a changelog entry in your commit message and read the [changelog entries section](https://docs.gitlab.com/ee/development/changelog.html).
|
157
|
+
|
158
|
+
If you forget to set the changelog entry in your commit messages, you can also edit the release notes after it's being released.
|
159
|
+
|
160
|
+
### Steps to release
|
161
|
+
|
162
|
+
Use a `Release` merge request template and create a merge requet to update the version number in `version.rb`, and get the merge request merged by a maintainer.
|
163
|
+
|
164
|
+
This will then be packaged into a gem and pushed to [rubygems.org](https://rubygems.org) by the CI/CD.
|
165
|
+
|
166
|
+
For example: [Bump version to 3.4.3](https://gitlab.com/gitlab-org/ruby/gems/gitlab-dangerfiles/-/merge_requests/149).
|
26
167
|
|
27
168
|
## License
|
28
169
|
|
data/exe/generate-test-session
CHANGED
@@ -11,7 +11,7 @@ params = {}
|
|
11
11
|
options = OptionParser.new do |opts|
|
12
12
|
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
13
13
|
|
14
|
-
opts.on('-i', '--input-files FILES', String, '
|
14
|
+
opts.on('-i', '--input-files FILES', String, 'RSpec report files (JSON or JUnit XML)') do |input_files|
|
15
15
|
params[:input_files] = input_files
|
16
16
|
end
|
17
17
|
|
@@ -34,6 +34,7 @@ options = OptionParser.new do |opts|
|
|
34
34
|
end
|
35
35
|
|
36
36
|
opts.on_tail('-h', '--help', 'Show the usage') do
|
37
|
+
puts "Purpose: Generate test session report based on RSpec report files (JSON or JUnit XML)"
|
37
38
|
puts opts
|
38
39
|
exit
|
39
40
|
end
|
data/exe/post-to-slack
CHANGED
@@ -23,7 +23,7 @@ options = OptionParser.new do |opts|
|
|
23
23
|
params[:message] = message
|
24
24
|
end
|
25
25
|
|
26
|
-
opts.on('-t', '--include-summary-table FILES', String, '
|
26
|
+
opts.on('-t', '--include-summary-table FILES', String, 'Add a test summary table based on RSpec report files (JUnit XML)') do |files|
|
27
27
|
params[:message] += "\n\n#{GitlabQuality::TestTooling::SummaryTable.create(input_files: files)}"
|
28
28
|
end
|
29
29
|
|
@@ -42,6 +42,7 @@ options = OptionParser.new do |opts|
|
|
42
42
|
end
|
43
43
|
|
44
44
|
opts.on_tail('-h', '--help', 'Show the usage') do
|
45
|
+
puts "Purpose: Post a message to Slack, and optionally add a test summary table based on RSpec report files (JUnit XML)"
|
45
46
|
puts opts
|
46
47
|
exit
|
47
48
|
end
|
data/exe/prepare-stage-reports
CHANGED
@@ -11,8 +11,7 @@ params = {}
|
|
11
11
|
options = OptionParser.new do |opts|
|
12
12
|
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
13
13
|
|
14
|
-
opts.on('-j', '--junit-files FILES', String,
|
15
|
-
'Prepare separate reports for each Stage from the provided JUnit XML files') do |junit_files|
|
14
|
+
opts.on('-j', '--junit-files FILES', String, 'RSpec report files (JUnit XML)') do |junit_files|
|
16
15
|
params[:junit_files] = junit_files
|
17
16
|
end
|
18
17
|
|
@@ -23,6 +22,7 @@ options = OptionParser.new do |opts|
|
|
23
22
|
end
|
24
23
|
|
25
24
|
opts.on_tail('-h', '--help', 'Show the usage') do
|
25
|
+
puts "Purpose: Prepare separate reports for each DevOps stage from the provided RSpec report files (JUnit XML)"
|
26
26
|
puts opts
|
27
27
|
exit
|
28
28
|
end
|
data/exe/relate-failure-issue
CHANGED
@@ -11,8 +11,7 @@ params = {}
|
|
11
11
|
options = OptionParser.new do |opts|
|
12
12
|
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
13
13
|
|
14
|
-
opts.on('-i', '--input-files FILES', String,
|
15
|
-
'Relate test failures to failure issues from RSpec JSON files') do |input_files|
|
14
|
+
opts.on('-i', '--input-files FILES', String, 'RSpec report files (JSON or JUnit XML)') do |input_files|
|
16
15
|
params[:input_files] = input_files
|
17
16
|
end
|
18
17
|
|
@@ -33,7 +32,7 @@ options = OptionParser.new do |opts|
|
|
33
32
|
params[:system_logs] = system_log_files
|
34
33
|
end
|
35
34
|
|
36
|
-
opts.on('--dry-run', "Perform a dry-run (don't create or update issues
|
35
|
+
opts.on('--dry-run', "Perform a dry-run (don't create or update issues)") do
|
37
36
|
params[:dry_run] = true
|
38
37
|
end
|
39
38
|
|
@@ -44,6 +43,7 @@ options = OptionParser.new do |opts|
|
|
44
43
|
end
|
45
44
|
|
46
45
|
opts.on_tail('-h', '--help', 'Show the usage') do
|
46
|
+
puts "Purpose: Relate test failures to failure issues from RSpec report files (JSON or JUnit XML)"
|
47
47
|
puts opts
|
48
48
|
exit
|
49
49
|
end
|
data/exe/report-results
CHANGED
@@ -11,8 +11,7 @@ params = {}
|
|
11
11
|
options = OptionParser.new do |opts|
|
12
12
|
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
13
13
|
|
14
|
-
opts.on('-i', '--input-files FILES', String,
|
15
|
-
'Report test results from JUnit or JSON files in GitLab test cases and results issues') do |input_files|
|
14
|
+
opts.on('-i', '--input-files FILES', String, 'RSpec report files (JSON or JUnit XML)') do |input_files|
|
16
15
|
params[:input_files] = input_files
|
17
16
|
end
|
18
17
|
|
@@ -41,6 +40,7 @@ options = OptionParser.new do |opts|
|
|
41
40
|
end
|
42
41
|
|
43
42
|
opts.on_tail('-h', '--help', 'Show the usage') do
|
43
|
+
puts "Purpose: Report test results from RSpec report files (JSON or JUnit XML) in GitLab test cases and result issues"
|
44
44
|
puts opts
|
45
45
|
exit
|
46
46
|
end
|
data/exe/update-screenshot-paths
CHANGED
@@ -11,9 +11,8 @@ params = {}
|
|
11
11
|
options = OptionParser.new do |opts|
|
12
12
|
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
13
13
|
|
14
|
-
opts.on('--
|
15
|
-
|
16
|
-
params[:junit_files] = junit_files
|
14
|
+
opts.on('-i', '--input-files FILES', String, 'RSpec report files (JSON or JUnit XML)') do |input_files|
|
15
|
+
params[:junit_files] = input_files
|
17
16
|
end
|
18
17
|
|
19
18
|
opts.on_tail('-v', '--version', 'Show the version') do
|
@@ -23,6 +22,7 @@ options = OptionParser.new do |opts|
|
|
23
22
|
end
|
24
23
|
|
25
24
|
opts.on_tail('-h', '--help', 'Show the usage') do
|
25
|
+
puts "Purpose: Update the path to screenshots to container's host from RSpec report files (JSON or JUnit XML)"
|
26
26
|
puts opts
|
27
27
|
exit
|
28
28
|
end
|
@@ -23,7 +23,7 @@ module GitlabQuality
|
|
23
23
|
module TestTooling
|
24
24
|
# The GitLab client is used for API access: https://github.com/NARKOZ/gitlab
|
25
25
|
class GitlabIssueClient
|
26
|
-
|
26
|
+
REPORTER_ACCESS_LEVEL = 20
|
27
27
|
RETRY_BACK_OFF_DELAY = 60
|
28
28
|
MAX_RETRY_ATTEMPTS = 3
|
29
29
|
|
@@ -40,7 +40,7 @@ module GitlabQuality
|
|
40
40
|
user = Gitlab.user
|
41
41
|
member = Gitlab.team_member(project, user.id)
|
42
42
|
|
43
|
-
abort_not_permitted if member.access_level <
|
43
|
+
abort_not_permitted if member.access_level < REPORTER_ACCESS_LEVEL
|
44
44
|
end
|
45
45
|
rescue Gitlab::Error::NotFound
|
46
46
|
abort_not_permitted
|
@@ -5,7 +5,6 @@ require 'nokogiri'
|
|
5
5
|
module GitlabQuality
|
6
6
|
module TestTooling
|
7
7
|
module Report
|
8
|
-
# Create a new JUnit report file for each Stage, containing tests from that Stage alone
|
9
8
|
class PrepareStageReports
|
10
9
|
EXTRACT_STAGE_FROM_TEST_FILE_REGEX = %r{(?:api|browser_ui)/(?:[0-9]+_)?(?<stage>[_\w]+)/}i
|
11
10
|
|