gitlab_quality-test_tooling 0.8.2 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.tool-versions +1 -0
- data/Gemfile.lock +1 -1
- data/README.md +12 -0
- data/exe/slow-test-issues +50 -0
- data/lib/gitlab_quality/test_tooling/report/concerns/group_and_category_labels.rb +24 -0
- data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb +1 -12
- data/lib/gitlab_quality/test_tooling/report/report_as_issue.rb +3 -3
- data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb +95 -0
- data/lib/gitlab_quality/test_tooling/test_results/test_result.rb +12 -0
- data/lib/gitlab_quality/test_tooling/version.rb +1 -1
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a3472eb65666a6691797e9e0f0332a1773bb6babe898245a375bd9976523866e
|
4
|
+
data.tar.gz: 6372e6df38afb56bbe300b39d25a65cb7cdd205765ded99cab3745f4fbe7d812
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 899483550e08f6f873d137d6a1bc90620ed58eff093827e76e5b4c92de8b7b1f95bba60181d2b726d79bf91b08b627cc93fb3817662dff73db8ba46e81557aad
|
7
|
+
data.tar.gz: 6b21fae0eb76efa44dd868113d22d36c30a9f5d06b5bb1b625dd42129a463a24ae77abf4edf6df05d06800b29261116492cb39aa892490aaa22cfca9ce442aa4
|
data/.tool-versions
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ruby 3.0.5
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -121,6 +121,18 @@ Usage: exe/update-screenshot-paths [options]
|
|
121
121
|
-h, --help Show the usage
|
122
122
|
```
|
123
123
|
|
124
|
+
### `slow-test-issues`
|
125
|
+
|
126
|
+
```shell
|
127
|
+
Purpose: Create slow test issues from JSON RSpec report files
|
128
|
+
Usage: exe/slow-test-issue [options]
|
129
|
+
-i, --input-files INPUT_FILES JSON RSpec report files JSON
|
130
|
+
-p, --project PROJECT Can be an integer or a group/project string
|
131
|
+
-t, --token TOKEN A valid access token with `api` scope and Maintainer permission in PROJECT
|
132
|
+
--dry-run Perform a dry-run (don't create issues)
|
133
|
+
-v, --version Show the version
|
134
|
+
-h, --help Show the usage
|
135
|
+
```
|
124
136
|
## Development
|
125
137
|
|
126
138
|
### Initial setup
|
@@ -0,0 +1,50 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require "bundler/setup"
|
5
|
+
require "optparse"
|
6
|
+
|
7
|
+
require_relative "../lib/gitlab_quality/test_tooling"
|
8
|
+
|
9
|
+
params = {}
|
10
|
+
|
11
|
+
options = OptionParser.new do |opts|
|
12
|
+
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
13
|
+
|
14
|
+
opts.on('-i', '--input-files INPUT_FILES', String, 'JSON RSpec report files JSON') do |input_files|
|
15
|
+
params[:input_files] = input_files
|
16
|
+
end
|
17
|
+
|
18
|
+
opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
|
19
|
+
params[:project] = project
|
20
|
+
end
|
21
|
+
|
22
|
+
opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
|
23
|
+
params[:token] = token
|
24
|
+
end
|
25
|
+
|
26
|
+
opts.on('--dry-run', "Perform a dry-run (don't create issues)") do
|
27
|
+
params[:dry_run] = true
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on_tail('-v', '--version', 'Show the version') do
|
31
|
+
require_relative "../lib/gitlab_quality/test_tooling/version"
|
32
|
+
puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
|
33
|
+
exit
|
34
|
+
end
|
35
|
+
|
36
|
+
opts.on_tail('-h', '--help', 'Show the usage') do
|
37
|
+
puts "Purpose: Create slow test issues from JSON RSpec report files"
|
38
|
+
puts opts
|
39
|
+
exit
|
40
|
+
end
|
41
|
+
|
42
|
+
opts.parse(ARGV)
|
43
|
+
end
|
44
|
+
|
45
|
+
if params.any?
|
46
|
+
GitlabQuality::TestTooling::Report::SlowTestIssue.new(**params).invoke!
|
47
|
+
else
|
48
|
+
puts options
|
49
|
+
exit 1
|
50
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module Report
|
6
|
+
module Concerns
|
7
|
+
module GroupAndCategoryLabels
|
8
|
+
def labels_inference
|
9
|
+
@labels_inference ||= GitlabQuality::TestTooling::LabelsInference.new
|
10
|
+
end
|
11
|
+
|
12
|
+
def new_issue_labels(test)
|
13
|
+
puts " => [DEBUG] product_group: #{test.product_group}; feature_category: #{test.feature_category}"
|
14
|
+
|
15
|
+
new_labels = self.class::NEW_ISSUE_LABELS +
|
16
|
+
labels_inference.infer_labels_from_product_group(test.product_group) +
|
17
|
+
labels_inference.infer_labels_from_feature_category(test.feature_category)
|
18
|
+
up_to_date_labels(test: test, new_labels: new_labels)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -16,6 +16,7 @@ module GitlabQuality
|
|
16
16
|
# - Add the failed job to the issue description, and update labels
|
17
17
|
class RelateFailureIssue < ReportAsIssue
|
18
18
|
include Concerns::FindSetDri
|
19
|
+
include Concerns::GroupAndCategoryLabels
|
19
20
|
|
20
21
|
DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
|
21
22
|
SYSTEMIC_EXCEPTIONS_THRESHOLD = 10
|
@@ -313,18 +314,6 @@ module GitlabQuality
|
|
313
314
|
"1. #{Time.new.utc.strftime('%F')}: #{test.ci_job_url} (#{ENV.fetch('CI_PIPELINE_URL', 'pipeline url is missing')})"
|
314
315
|
end
|
315
316
|
|
316
|
-
def labels_inference
|
317
|
-
@labels_inference ||= GitlabQuality::TestTooling::LabelsInference.new
|
318
|
-
end
|
319
|
-
|
320
|
-
def new_issue_labels(test)
|
321
|
-
puts " => [DEBUG] product_group: #{test.product_group}; feature_category: #{test.feature_category}"
|
322
|
-
new_labels = NEW_ISSUE_LABELS +
|
323
|
-
labels_inference.infer_labels_from_product_group(test.product_group) +
|
324
|
-
labels_inference.infer_labels_from_feature_category(test.feature_category)
|
325
|
-
up_to_date_labels(test: test, new_labels: new_labels)
|
326
|
-
end
|
327
|
-
|
328
317
|
def up_to_date_labels(test:, issue: nil, new_labels: Set.new)
|
329
318
|
(Set.new(base_issue_labels) + (super << pipeline_name_label)).to_a
|
330
319
|
end
|
@@ -30,7 +30,7 @@ module GitlabQuality
|
|
30
30
|
raise NotImplementedError
|
31
31
|
end
|
32
32
|
|
33
|
-
def
|
33
|
+
def test_hash(test)
|
34
34
|
OpenSSL::Digest::SHA256.hexdigest(test.file + test.name)
|
35
35
|
end
|
36
36
|
|
@@ -42,7 +42,7 @@ module GitlabQuality
|
|
42
42
|
| ------ | ------ |
|
43
43
|
| File | #{test_file_link(test)} |
|
44
44
|
| Description | `#{test.name}` |
|
45
|
-
| Hash | `#{
|
45
|
+
| Hash | `#{test_hash(test)}` |
|
46
46
|
#{"| Test case | #{test.testcase} |" if test.testcase}
|
47
47
|
DESCRIPTION
|
48
48
|
end
|
@@ -50,7 +50,7 @@ module GitlabQuality
|
|
50
50
|
def test_file_link(test)
|
51
51
|
path_prefix = test.file.start_with?('qa/') ? 'qa/' : ''
|
52
52
|
|
53
|
-
"[`#{path_prefix}#{test.file}`](#{FILE_BASE_URL}#{path_prefix}#{test.file})"
|
53
|
+
"[`#{path_prefix}#{test.file}`](#{FILE_BASE_URL}#{path_prefix}#{test.file}##{test.line_number})"
|
54
54
|
end
|
55
55
|
|
56
56
|
def new_issue_labels(_test)
|
@@ -0,0 +1,95 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module Report
|
6
|
+
# Uses the API to create GitLab issues for slow tests
|
7
|
+
#
|
8
|
+
# - Takes the JSON test reports like rspec-*.json`
|
9
|
+
# - Takes a project where slow issues should be created
|
10
|
+
# - Find issue by title (with test description or test file)
|
11
|
+
# - Add test metadata, duration to the issue with group and category labels
|
12
|
+
class SlowTestIssue < ReportAsIssue
|
13
|
+
include Concerns::FindSetDri
|
14
|
+
include Concerns::GroupAndCategoryLabels
|
15
|
+
|
16
|
+
NEW_ISSUE_LABELS = Set.new(%w[test type::maintenance maintenance::performance priority::3 severity::3]).freeze
|
17
|
+
SEARCH_LABELS = %w[test maintenance::performance].freeze
|
18
|
+
|
19
|
+
MultipleIssuesFound = Class.new(StandardError)
|
20
|
+
|
21
|
+
TestLevelSpecification = Struct.new(:regex, :max_duration)
|
22
|
+
|
23
|
+
OTHER_TESTS_MAX_DURATION = 45.40 # seconds
|
24
|
+
|
25
|
+
TEST_LEVEL_SPECIFICATIONS = [
|
26
|
+
TestLevelSpecification.new(%r{/features/}, 50.13),
|
27
|
+
TestLevelSpecification.new(%r{/controllers|requests/}, 19.20),
|
28
|
+
TestLevelSpecification.new(%r{/lib/}, 27.12)
|
29
|
+
].freeze
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def run!
|
34
|
+
puts "Reporting slow tests in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
35
|
+
|
36
|
+
TestResults::Builder.new(files).test_results_per_file do |test_results|
|
37
|
+
puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
|
38
|
+
|
39
|
+
test_results.each do |test|
|
40
|
+
create_slow_issue(test) if should_create_slow_issue?(test)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def new_issue_title(test)
|
46
|
+
"Slow test in #{super}"
|
47
|
+
end
|
48
|
+
|
49
|
+
def new_issue_description(test)
|
50
|
+
super + [
|
51
|
+
"\n### Slow test",
|
52
|
+
"Slow tests detected, see guides for more details and how to improve them:",
|
53
|
+
"- [Top slow tests](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#top-slow-tests)",
|
54
|
+
"- [Test speed](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-speed)",
|
55
|
+
"**Duration**: #{test.run_time} seconds"
|
56
|
+
].compact.join("\n\n")
|
57
|
+
end
|
58
|
+
|
59
|
+
def create_slow_issue(test)
|
60
|
+
puts " => Finding existing issues for slow test '#{test.name}' (run time: #{test.run_time} seconds)..."
|
61
|
+
|
62
|
+
issue = find_issue(test)
|
63
|
+
|
64
|
+
puts " => Existing issue link #{issue['web_url']}" if issue.present?
|
65
|
+
|
66
|
+
create_issue(test) unless issue.present?
|
67
|
+
rescue MultipleIssuesFound => e
|
68
|
+
warn(e.message)
|
69
|
+
end
|
70
|
+
|
71
|
+
def find_issue(test)
|
72
|
+
search_labels = SEARCH_LABELS
|
73
|
+
|
74
|
+
gitlab.find_issues(options: { state: 'opened', labels: search_labels.to_a }).find do |issue|
|
75
|
+
issue_title = issue.title.strip
|
76
|
+
issue_title.include?(test.name) || issue_title.include?(partial_file_path(test.file))
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def should_create_slow_issue?(test)
|
81
|
+
test.run_time > max_duration_for_test(test)
|
82
|
+
end
|
83
|
+
|
84
|
+
def max_duration_for_test(test)
|
85
|
+
test_level_specification = TEST_LEVEL_SPECIFICATIONS.find do |test_level_specification|
|
86
|
+
test.example_id =~ test_level_specification.regex
|
87
|
+
end
|
88
|
+
return OTHER_TESTS_MAX_DURATION unless test_level_specification
|
89
|
+
|
90
|
+
test_level_specification.max_duration
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -115,6 +115,18 @@ module GitlabQuality
|
|
115
115
|
report['feature_category']
|
116
116
|
end
|
117
117
|
|
118
|
+
def run_time
|
119
|
+
report['run_time'].to_f.round(2)
|
120
|
+
end
|
121
|
+
|
122
|
+
def example_id
|
123
|
+
report['id']
|
124
|
+
end
|
125
|
+
|
126
|
+
def line_number
|
127
|
+
report['line_number']
|
128
|
+
end
|
129
|
+
|
118
130
|
private
|
119
131
|
|
120
132
|
# rubocop:disable Metrics/AbcSize
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab_quality-test_tooling
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- GitLab Quality
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-06-
|
11
|
+
date: 2023-06-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|
@@ -323,6 +323,7 @@ executables:
|
|
323
323
|
- prepare-stage-reports
|
324
324
|
- relate-failure-issue
|
325
325
|
- report-results
|
326
|
+
- slow-test-issues
|
326
327
|
- update-screenshot-paths
|
327
328
|
extensions: []
|
328
329
|
extra_rdoc_files: []
|
@@ -331,6 +332,7 @@ files:
|
|
331
332
|
- ".rubocop.yml"
|
332
333
|
- ".rubocop_todo.yml"
|
333
334
|
- ".ruby-version"
|
335
|
+
- ".tool-versions"
|
334
336
|
- CODE_OF_CONDUCT.md
|
335
337
|
- CONTRIBUTING.md
|
336
338
|
- Dangerfile
|
@@ -345,6 +347,7 @@ files:
|
|
345
347
|
- exe/prepare-stage-reports
|
346
348
|
- exe/relate-failure-issue
|
347
349
|
- exe/report-results
|
350
|
+
- exe/slow-test-issues
|
348
351
|
- exe/update-screenshot-paths
|
349
352
|
- lefthook.yml
|
350
353
|
- lib/gitlab_quality/test_tooling.rb
|
@@ -352,6 +355,7 @@ files:
|
|
352
355
|
- lib/gitlab_quality/test_tooling/gitlab_issue_dry_client.rb
|
353
356
|
- lib/gitlab_quality/test_tooling/labels_inference.rb
|
354
357
|
- lib/gitlab_quality/test_tooling/report/concerns/find_set_dri.rb
|
358
|
+
- lib/gitlab_quality/test_tooling/report/concerns/group_and_category_labels.rb
|
355
359
|
- lib/gitlab_quality/test_tooling/report/concerns/results_reporter.rb
|
356
360
|
- lib/gitlab_quality/test_tooling/report/concerns/utils.rb
|
357
361
|
- lib/gitlab_quality/test_tooling/report/generate_test_session.rb
|
@@ -361,6 +365,7 @@ files:
|
|
361
365
|
- lib/gitlab_quality/test_tooling/report/report_results.rb
|
362
366
|
- lib/gitlab_quality/test_tooling/report/results_in_issues.rb
|
363
367
|
- lib/gitlab_quality/test_tooling/report/results_in_test_cases.rb
|
368
|
+
- lib/gitlab_quality/test_tooling/report/slow_test_issue.rb
|
364
369
|
- lib/gitlab_quality/test_tooling/report/update_screenshot_path.rb
|
365
370
|
- lib/gitlab_quality/test_tooling/runtime/env.rb
|
366
371
|
- lib/gitlab_quality/test_tooling/runtime/logger.rb
|