gitlab_quality-test_tooling 3.4.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +0 -14
- data/exe/test-coverage +13 -2
- data/lib/gitlab_quality/test_tooling/code_coverage/click_house/category_owners_table.rb +9 -0
- data/lib/gitlab_quality/test_tooling/report/results_in_test_cases.rb +2 -4
- data/lib/gitlab_quality/test_tooling/test_meta/processor/meta_processor.rb +1 -1
- data/lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb +4 -4
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/test_metrics.rb +20 -0
- data/lib/gitlab_quality/test_tooling/test_result/base_test_result.rb +1 -0
- data/lib/gitlab_quality/test_tooling/version.rb +1 -1
- metadata +2 -8
- data/exe/existing-test-health-issue +0 -59
- data/exe/generate-test-session +0 -70
- data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb +0 -288
- data/lib/gitlab_quality/test_tooling/report/test_health_issue_finder.rb +0 -79
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0afeda440d33c3cb9e1e81adf43ce462d12b604cffdf96b04dbe83a9a4d1f853
|
|
4
|
+
data.tar.gz: 1a638b0a0b409ed28cb94da821a59f73967b6ae55cad7e87552a096b9c5c525e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5bece6881a2ab2506f4d22b7da49b976af027a0f7d93c276d704f37fb52091ddde319232db0ccca1b717141267bbd5e283f30796a8bc06b14dbbbd38115327ca
|
|
7
|
+
data.tar.gz: 7136b103704d811fcdaee9a44ee815775d2eb0411934654131d6478a8d2b62c3225a94a5602de169284cd3121a1b464bcbcf65570a5f7f923add0cdb0b5b386a
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
|
@@ -185,20 +185,6 @@ Usage: exe/failed-test-issues [options]
|
|
|
185
185
|
-h, --help Show the usage
|
|
186
186
|
```
|
|
187
187
|
|
|
188
|
-
### `exe/existing-test-health-issue`
|
|
189
|
-
|
|
190
|
-
```shell
|
|
191
|
-
Purpose: Checks whether tests coming from the rspec JSON report files has an existing test health issue opened.
|
|
192
|
-
Usage: exe/existing-test-health-issue [options]
|
|
193
|
-
-i, --input-files INPUT_FILES JSON rspec-retry report files
|
|
194
|
-
-p, --project PROJECT Can be an integer or a group/project string
|
|
195
|
-
-t, --token TOKEN A valid access token with `api` scope and Maintainer permission in PROJECT
|
|
196
|
-
--health-problem-type PROBLEM_TYPE
|
|
197
|
-
Look for the given health problem type (failures, pass-after-retry, slow)
|
|
198
|
-
-v, --version Show the version
|
|
199
|
-
-h, --help Show the usage
|
|
200
|
-
```
|
|
201
|
-
|
|
202
188
|
### `exe/detect-infrastructure-failures`
|
|
203
189
|
|
|
204
190
|
```shell
|
data/exe/test-coverage
CHANGED
|
@@ -55,6 +55,10 @@ options = OptionParser.new do |opts|
|
|
|
55
55
|
params[:clickhouse_username] = username
|
|
56
56
|
end
|
|
57
57
|
|
|
58
|
+
opts.on('--clickhouse-shared-database DATABASE', 'ClickHouse shared database name (default: shared)') do |database|
|
|
59
|
+
params[:clickhouse_shared_database] = database
|
|
60
|
+
end
|
|
61
|
+
|
|
58
62
|
opts.on('--responsibility-patterns PATH', 'Path to YAML file with responsibility classification patterns') do |path|
|
|
59
63
|
params[:responsibility_patterns] = path
|
|
60
64
|
end
|
|
@@ -166,7 +170,14 @@ if params.any? && (required_params - params.keys).none?
|
|
|
166
170
|
password: clickhouse_password
|
|
167
171
|
}
|
|
168
172
|
|
|
169
|
-
|
|
173
|
+
shared_clickhouse_data = {
|
|
174
|
+
url: params[:clickhouse_url],
|
|
175
|
+
database: params[:clickhouse_shared_database] || 'shared',
|
|
176
|
+
username: params[:clickhouse_username],
|
|
177
|
+
password: clickhouse_password
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
category_owners_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CategoryOwnersTable.new(**shared_clickhouse_data)
|
|
170
181
|
coverage_metrics_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::CoverageMetricsTable.new(**clickhouse_data)
|
|
171
182
|
|
|
172
183
|
if ENV['CLICKHOUSE_PUSH_CATEGORY_DATA'] == 'true'
|
|
@@ -178,7 +189,7 @@ if params.any? && (required_params - params.keys).none?
|
|
|
178
189
|
|
|
179
190
|
# Export test-to-file mappings
|
|
180
191
|
test_file_mapping_data = GitlabQuality::TestTooling::CodeCoverage::TestFileMappingData.new(test_to_sources)
|
|
181
|
-
test_file_mappings_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestFileMappingsTable.new(**
|
|
192
|
+
test_file_mappings_table = GitlabQuality::TestTooling::CodeCoverage::ClickHouse::TestFileMappingsTable.new(**shared_clickhouse_data)
|
|
182
193
|
test_file_mappings_table.push(test_file_mapping_data.as_db_table)
|
|
183
194
|
else
|
|
184
195
|
puts "Missing argument(s). Required arguments are: #{required_params}\nPassed arguments are: #{params}\n"
|
|
@@ -11,6 +11,8 @@ module GitlabQuality
|
|
|
11
11
|
|
|
12
12
|
MissingMappingError = Class.new(StandardError)
|
|
13
13
|
|
|
14
|
+
KNOWN_UNOWNED = %w[shared not_owned tooling].freeze
|
|
15
|
+
|
|
14
16
|
def truncate
|
|
15
17
|
logger.debug("#{LOG_PREFIX} Truncating table #{full_table_name} ...")
|
|
16
18
|
|
|
@@ -24,6 +26,13 @@ module GitlabQuality
|
|
|
24
26
|
# @param feature_category_name [String] the feature_category name
|
|
25
27
|
# @return [Hash]
|
|
26
28
|
def owners(feature_category_name)
|
|
29
|
+
if KNOWN_UNOWNED.include?(feature_category_name)
|
|
30
|
+
logger.info(
|
|
31
|
+
"#{LOG_PREFIX} #{feature_category_name} is a known feature category without owner..."
|
|
32
|
+
)
|
|
33
|
+
return {}
|
|
34
|
+
end
|
|
35
|
+
|
|
27
36
|
records.fetch(feature_category_name)
|
|
28
37
|
rescue KeyError
|
|
29
38
|
raise(MissingMappingError, "Feature category '#{feature_category_name}' not found in table '#{table_name}'")
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'erb'
|
|
4
|
-
|
|
5
3
|
module GitlabQuality
|
|
6
4
|
module TestTooling
|
|
7
5
|
module Report
|
|
@@ -68,12 +66,12 @@ module GitlabQuality
|
|
|
68
66
|
end
|
|
69
67
|
|
|
70
68
|
def execution_graph_section(test)
|
|
71
|
-
|
|
69
|
+
formatted_path = CGI.escape(test.relative_file)
|
|
72
70
|
|
|
73
71
|
<<~MKDOWN.strip
|
|
74
72
|
### Executions
|
|
75
73
|
|
|
76
|
-
[Spec metrics on all environments](https://dashboards.
|
|
74
|
+
[Spec metrics on all environments](https://dashboards.devex.gitlab.net/d/739c1bdd-a436-452b-bddc-fccb4d055768/single-test-overview?var-file_path=#{formatted_path})
|
|
77
75
|
MKDOWN
|
|
78
76
|
end
|
|
79
77
|
|
|
@@ -45,7 +45,7 @@ module GitlabQuality
|
|
|
45
45
|
commits.each_with_index.map do |(changed_line_number, spec), index|
|
|
46
46
|
<<~MARKDOWN
|
|
47
47
|
#{index + 1}. [`#{spec['name']}`](https://gitlab.com/#{context.project}/-/blob/#{context.ref}/#{spec['file_path']}#L#{changed_line_number.to_i + 1})
|
|
48
|
-
| [Testcase](#{spec['testcase']}) | [Spec metrics](#{context.single_spec_metrics_link(spec['
|
|
48
|
+
| [Testcase](#{spec['testcase']}) | [Spec metrics](#{context.single_spec_metrics_link(spec['file_path'])})
|
|
49
49
|
#{failure_issue_text(spec)}
|
|
50
50
|
MARKDOWN
|
|
51
51
|
end.join("\n")
|
|
@@ -348,11 +348,11 @@ module GitlabQuality
|
|
|
348
348
|
|
|
349
349
|
# Returns the link to the Grafana dashboard for single spec metrics
|
|
350
350
|
#
|
|
351
|
-
# @param [String]
|
|
351
|
+
# @param [String] file_path the full path of spec
|
|
352
352
|
# @return [String]
|
|
353
|
-
def single_spec_metrics_link(
|
|
354
|
-
base_url = "https://dashboards.
|
|
355
|
-
base_url + CGI.escape(
|
|
353
|
+
def single_spec_metrics_link(file_path)
|
|
354
|
+
base_url = "https://dashboards.devex.gitlab.net/d/739c1bdd-a436-452b-bddc-fccb4d055768/single-test-overview?var-file_path="
|
|
355
|
+
base_url + CGI.escape(file_path)
|
|
356
356
|
end
|
|
357
357
|
|
|
358
358
|
# Returns any test description string within single or double quotes
|
|
@@ -51,6 +51,7 @@ module GitlabQuality
|
|
|
51
51
|
exception_classes: exception_classes.map { |e| e.class.to_s }.uniq,
|
|
52
52
|
failure_exception: failure_exception,
|
|
53
53
|
quarantined: quarantined?,
|
|
54
|
+
quarantine_issue_url: quarantine_issue_url || "",
|
|
54
55
|
feature_category: example.metadata[:feature_category] || "",
|
|
55
56
|
test_retried: config.test_retried_proc.call(example),
|
|
56
57
|
run_type: run_type,
|
|
@@ -98,6 +99,25 @@ module GitlabQuality
|
|
|
98
99
|
example.execution_result.status == :pending
|
|
99
100
|
end
|
|
100
101
|
|
|
102
|
+
# Extract quarantine issue URL from metadata
|
|
103
|
+
#
|
|
104
|
+
# @return [String, nil]
|
|
105
|
+
def quarantine_issue_url
|
|
106
|
+
return nil unless example.metadata.key?(:quarantine)
|
|
107
|
+
|
|
108
|
+
metadata = example.metadata[:quarantine]
|
|
109
|
+
case metadata
|
|
110
|
+
when String
|
|
111
|
+
# Direct URL: quarantine: 'https://gitlab.com/.../issues/123'
|
|
112
|
+
metadata if metadata.start_with?('http')
|
|
113
|
+
when Hash
|
|
114
|
+
# Hash format: quarantine: { issue: 'https://...', reason: '...' }
|
|
115
|
+
issue = metadata[:issue] || metadata['issue']
|
|
116
|
+
# Handle array of URLs (take the first one)
|
|
117
|
+
issue.is_a?(Array) ? issue.first : issue
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
101
121
|
# Base ci job name
|
|
102
122
|
#
|
|
103
123
|
# @return [String]
|
|
@@ -14,6 +14,7 @@ module GitlabQuality
|
|
|
14
14
|
"could not be found (502)",
|
|
15
15
|
"Error reference number: 502",
|
|
16
16
|
"(502): `GitLab is not responding`",
|
|
17
|
+
"(502): `502 Bad Gateway`",
|
|
17
18
|
"<head><title>502 Bad Gateway</title></head>",
|
|
18
19
|
"14:connections to all backends failing",
|
|
19
20
|
"gitlab_canary=true cookie was set in browser but 'Next' badge was not shown on UI"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gitlab_quality-test_tooling
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 3.
|
|
4
|
+
version: 3.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- GitLab Quality
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-01-
|
|
11
|
+
date: 2026-01-23 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: climate_control
|
|
@@ -416,12 +416,10 @@ email:
|
|
|
416
416
|
executables:
|
|
417
417
|
- detect-infrastructure-failures
|
|
418
418
|
- epic-readiness-notification
|
|
419
|
-
- existing-test-health-issue
|
|
420
419
|
- failed-test-issues
|
|
421
420
|
- feature-readiness-checklist
|
|
422
421
|
- feature-readiness-evaluation
|
|
423
422
|
- flaky-test-issues
|
|
424
|
-
- generate-test-session
|
|
425
423
|
- knapsack-report-issues
|
|
426
424
|
- post-to-slack
|
|
427
425
|
- prepare-stage-reports
|
|
@@ -451,12 +449,10 @@ files:
|
|
|
451
449
|
- Rakefile
|
|
452
450
|
- exe/detect-infrastructure-failures
|
|
453
451
|
- exe/epic-readiness-notification
|
|
454
|
-
- exe/existing-test-health-issue
|
|
455
452
|
- exe/failed-test-issues
|
|
456
453
|
- exe/feature-readiness-checklist
|
|
457
454
|
- exe/feature-readiness-evaluation
|
|
458
455
|
- exe/flaky-test-issues
|
|
459
|
-
- exe/generate-test-session
|
|
460
456
|
- exe/knapsack-report-issues
|
|
461
457
|
- exe/post-to-slack
|
|
462
458
|
- exe/prepare-stage-reports
|
|
@@ -526,7 +522,6 @@ files:
|
|
|
526
522
|
- lib/gitlab_quality/test_tooling/report/failed_test_issue.rb
|
|
527
523
|
- lib/gitlab_quality/test_tooling/report/feature_readiness/report_on_epic.rb
|
|
528
524
|
- lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb
|
|
529
|
-
- lib/gitlab_quality/test_tooling/report/generate_test_session.rb
|
|
530
525
|
- lib/gitlab_quality/test_tooling/report/group_issues/error_message_normalizer.rb
|
|
531
526
|
- lib/gitlab_quality/test_tooling/report/group_issues/error_pattern_matcher.rb
|
|
532
527
|
- lib/gitlab_quality/test_tooling/report/group_issues/failure_processor.rb
|
|
@@ -549,7 +544,6 @@ files:
|
|
|
549
544
|
- lib/gitlab_quality/test_tooling/report/results_in_issues.rb
|
|
550
545
|
- lib/gitlab_quality/test_tooling/report/results_in_test_cases.rb
|
|
551
546
|
- lib/gitlab_quality/test_tooling/report/slow_test_issue.rb
|
|
552
|
-
- lib/gitlab_quality/test_tooling/report/test_health_issue_finder.rb
|
|
553
547
|
- lib/gitlab_quality/test_tooling/report/update_screenshot_path.rb
|
|
554
548
|
- lib/gitlab_quality/test_tooling/runtime/env.rb
|
|
555
549
|
- lib/gitlab_quality/test_tooling/runtime/logger.rb
|
|
@@ -1,59 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ruby
|
|
2
|
-
# frozen_string_literal: true
|
|
3
|
-
|
|
4
|
-
require "bundler/setup"
|
|
5
|
-
require "optparse"
|
|
6
|
-
|
|
7
|
-
require_relative "../lib/gitlab_quality/test_tooling"
|
|
8
|
-
|
|
9
|
-
params = {}
|
|
10
|
-
HEALTH_PROBLEM_TYPES = GitlabQuality::TestTooling::Report::TestHealthIssueFinder::HEALTH_PROBLEM_TYPE_TO_LABEL.keys
|
|
11
|
-
|
|
12
|
-
options = OptionParser.new do |opts|
|
|
13
|
-
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
|
14
|
-
|
|
15
|
-
opts.on('-i', '--input-files INPUT_FILES', String, 'JSON rspec-retry report files') do |input_files|
|
|
16
|
-
params[:input_files] = input_files
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
|
|
20
|
-
params[:project] = project
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Maintainer permission in PROJECT') do |token|
|
|
24
|
-
params[:token] = token
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
opts.on("--health-problem-type PROBLEM_TYPE", String, "Look for the given health problem type (#{HEALTH_PROBLEM_TYPES.join(', ')})") do |value|
|
|
28
|
-
raise ArgumentError, "Invalid health problem type: #{value}. Valid options are: #{HEALTH_PROBLEM_TYPES.join(', ')}" unless HEALTH_PROBLEM_TYPES.include?(value)
|
|
29
|
-
|
|
30
|
-
params[:health_problem_type] = value
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
opts.on_tail('-v', '--version', 'Show the version') do
|
|
34
|
-
require_relative "../lib/gitlab_quality/test_tooling/version"
|
|
35
|
-
puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
|
|
36
|
-
exit
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
opts.on_tail('-h', '--help', 'Show the usage') do
|
|
40
|
-
puts "Purpose: Checks whether tests coming from the rspec JSON report files has an existing test health issue opened."
|
|
41
|
-
puts opts
|
|
42
|
-
exit
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
opts.parse(ARGV)
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
if params.any?
|
|
49
|
-
raise ArgumentError, "No health problem type given. Valid options are: #{HEALTH_PROBLEM_TYPES.join(', ')}" unless params.key?(:health_problem_type)
|
|
50
|
-
|
|
51
|
-
if GitlabQuality::TestTooling::Report::TestHealthIssueFinder.new(**params).found_existing_unhealthy_test_issue?
|
|
52
|
-
exit 0
|
|
53
|
-
else
|
|
54
|
-
exit 1
|
|
55
|
-
end
|
|
56
|
-
else
|
|
57
|
-
puts options
|
|
58
|
-
exit 1
|
|
59
|
-
end
|
data/exe/generate-test-session
DELETED
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ruby
|
|
2
|
-
# frozen_string_literal: true
|
|
3
|
-
|
|
4
|
-
require "bundler/setup"
|
|
5
|
-
require "optparse"
|
|
6
|
-
|
|
7
|
-
require_relative "../lib/gitlab_quality/test_tooling"
|
|
8
|
-
|
|
9
|
-
params = {}
|
|
10
|
-
|
|
11
|
-
options = OptionParser.new do |opts|
|
|
12
|
-
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
|
13
|
-
|
|
14
|
-
opts.on('-i', '--input-files INPUT_FILES', String, 'RSpec report files (JSON or JUnit XML)') do |input_files|
|
|
15
|
-
params[:input_files] = input_files
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
opts.on('-p', '--project PROJECT', String, 'Can be an integer or a group/project string') do |project|
|
|
19
|
-
params[:project] = project
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
opts.on('-t', '--token TOKEN', String, 'A valid access token with `api` scope and Reporter permission in PROJECT') do |token|
|
|
23
|
-
params[:token] = token
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
opts.on('-c', '--ci-project-token CI_PROJECT_TOKEN', String, 'A valid access token with `read_api` scope permission in current ENV["CI_PROJECT_ID"]') do |ci_project_token|
|
|
27
|
-
params[:ci_project_token] = ci_project_token
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
opts.on('-f', '--issue-url-file ISSUE_URL_FILE', 'Output the created test session issue URL') do |issue_url_file|
|
|
31
|
-
params[:issue_url_file] = issue_url_file
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
opts.on('--pipeline-stages STAGES', String, 'Comma-separated list of pipeline stages to include in test session issue') do |pipeline_stages|
|
|
35
|
-
params[:pipeline_stages] = pipeline_stages.split(',')
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
opts.on('--confidential', "Makes test session issue confidential") do
|
|
39
|
-
params[:confidential] = true
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
opts.on('--dry-run', "Perform a dry-run (don't create or update issues or test cases)") do
|
|
43
|
-
params[:dry_run] = true
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
opts.on_tail('-v', '--version', 'Show the version') do
|
|
47
|
-
require_relative "../lib/gitlab_quality/test_tooling/version"
|
|
48
|
-
puts "#{$PROGRAM_NAME} : #{GitlabQuality::TestTooling::VERSION}"
|
|
49
|
-
exit
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
opts.on_tail('-h', '--help', 'Show the usage') do
|
|
53
|
-
puts "Purpose: Generate test session report based on RSpec report files (JSON or JUnit XML)"
|
|
54
|
-
puts opts
|
|
55
|
-
exit
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
opts.parse(ARGV)
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
issue_url_file = params.delete(:issue_url_file)
|
|
62
|
-
|
|
63
|
-
if params.any?
|
|
64
|
-
issue_url = GitlabQuality::TestTooling::Report::GenerateTestSession.new(**params).invoke!
|
|
65
|
-
|
|
66
|
-
File.write(issue_url_file, issue_url) if issue_url_file && issue_url
|
|
67
|
-
else
|
|
68
|
-
puts options
|
|
69
|
-
exit 1
|
|
70
|
-
end
|
|
@@ -1,288 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'erb'
|
|
4
|
-
require 'date'
|
|
5
|
-
|
|
6
|
-
module GitlabQuality
|
|
7
|
-
module TestTooling
|
|
8
|
-
module Report
|
|
9
|
-
class GenerateTestSession < ReportAsIssue
|
|
10
|
-
def initialize(ci_project_token:, pipeline_stages: nil, **kwargs)
|
|
11
|
-
super
|
|
12
|
-
@ci_project_token = ci_project_token
|
|
13
|
-
@pipeline_stages = Set.new(pipeline_stages)
|
|
14
|
-
@issue_type = 'issue'
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
private
|
|
18
|
-
|
|
19
|
-
attr_reader :ci_project_token, :pipeline_stages
|
|
20
|
-
|
|
21
|
-
# rubocop:disable Metrics/AbcSize
|
|
22
|
-
def run!
|
|
23
|
-
puts "Generating test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
|
24
|
-
|
|
25
|
-
tests = Dir.glob(files).flat_map do |path|
|
|
26
|
-
puts "Loading tests in #{path}"
|
|
27
|
-
|
|
28
|
-
TestResults::JsonTestResults.new(path: path).to_a
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
tests = tests.select { |test| pipeline_stages.include? test.report["stage"] } unless pipeline_stages.empty?
|
|
32
|
-
|
|
33
|
-
issue = gitlab.create_issue(
|
|
34
|
-
title: "#{Time.now.to_date.iso8601} Test session report | #{Runtime::Env.qa_run_type}",
|
|
35
|
-
description: generate_description(tests),
|
|
36
|
-
labels: ['automation:bot-authored', 'E2E', 'triage report', pipeline_name_label, 'suppress-contributor-links'],
|
|
37
|
-
confidential: confidential
|
|
38
|
-
)
|
|
39
|
-
|
|
40
|
-
# Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/295493
|
|
41
|
-
unless Runtime::Env.qa_issue_url.to_s.empty?
|
|
42
|
-
gitlab.create_issue_note(
|
|
43
|
-
iid: issue.iid,
|
|
44
|
-
note: "/relate #{Runtime::Env.qa_issue_url}")
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
issue&.web_url # Issue isn't created in dry-run mode
|
|
48
|
-
end
|
|
49
|
-
# rubocop:enable Metrics/AbcSize
|
|
50
|
-
|
|
51
|
-
def generate_description(tests)
|
|
52
|
-
<<~MARKDOWN.rstrip
|
|
53
|
-
## Session summary
|
|
54
|
-
|
|
55
|
-
* Deploy version: #{Runtime::Env.deploy_version}
|
|
56
|
-
* Deploy environment: #{Runtime::Env.deploy_environment}
|
|
57
|
-
* Pipeline: #{Runtime::Env.pipeline_from_project_name} [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
|
|
58
|
-
#{generate_summary(tests: tests)}
|
|
59
|
-
|
|
60
|
-
#{generate_failed_jobs_listing}
|
|
61
|
-
|
|
62
|
-
#{generate_stages_listing(tests)}
|
|
63
|
-
|
|
64
|
-
#{generate_qa_issue_relation}
|
|
65
|
-
|
|
66
|
-
#{generate_link_to_dashboard}
|
|
67
|
-
MARKDOWN
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
def generate_summary(tests:, tests_by_status: nil)
|
|
71
|
-
tests_by_status ||= tests.group_by(&:status)
|
|
72
|
-
total = tests.size
|
|
73
|
-
passed = tests_by_status['passed']&.size || 0
|
|
74
|
-
failed = tests_by_status['failed']&.size || 0
|
|
75
|
-
others = total - passed - failed
|
|
76
|
-
|
|
77
|
-
<<~MARKDOWN.chomp
|
|
78
|
-
* Total #{total} tests
|
|
79
|
-
* Passed #{passed} tests
|
|
80
|
-
* Failed #{failed} tests
|
|
81
|
-
* #{others} other tests (usually skipped)
|
|
82
|
-
MARKDOWN
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
def generate_failed_jobs_listing
|
|
86
|
-
failed_jobs = fetch_pipeline_failed_jobs
|
|
87
|
-
listings = failed_jobs.filter_map do |job|
|
|
88
|
-
next if pipeline_stages.any? && !pipeline_stages.include?(job.stage)
|
|
89
|
-
|
|
90
|
-
allowed_to_fail = ' (allowed to fail)' if job.allow_failure
|
|
91
|
-
|
|
92
|
-
"* [#{job.name}](#{job.web_url})#{allowed_to_fail}"
|
|
93
|
-
end.join("\n")
|
|
94
|
-
|
|
95
|
-
<<~MARKDOWN.chomp if failed_jobs.any?
|
|
96
|
-
## Failed jobs
|
|
97
|
-
|
|
98
|
-
#{listings}
|
|
99
|
-
MARKDOWN
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
def generate_stages_listing(tests)
|
|
103
|
-
generate_tests_by_stage(tests).map do |stage, tests_for_stage|
|
|
104
|
-
tests_by_status = tests_for_stage.group_by(&:status)
|
|
105
|
-
|
|
106
|
-
<<~MARKDOWN.chomp
|
|
107
|
-
### #{stage&.capitalize || 'Unknown'}
|
|
108
|
-
|
|
109
|
-
#{generate_summary(
|
|
110
|
-
tests: tests_for_stage, tests_by_status: tests_by_status)}
|
|
111
|
-
|
|
112
|
-
#{generate_testcase_listing_by_status(
|
|
113
|
-
tests: tests_for_stage, tests_by_status: tests_by_status)}
|
|
114
|
-
MARKDOWN
|
|
115
|
-
end.join("\n\n")
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
def generate_tests_by_stage(tests)
|
|
119
|
-
# https://about.gitlab.com/handbook/product/product-categories/#devops-stages
|
|
120
|
-
ordering = %w[
|
|
121
|
-
manage
|
|
122
|
-
plan
|
|
123
|
-
create
|
|
124
|
-
verify
|
|
125
|
-
package
|
|
126
|
-
release
|
|
127
|
-
configure
|
|
128
|
-
monitor
|
|
129
|
-
secure
|
|
130
|
-
defend
|
|
131
|
-
growth
|
|
132
|
-
fulfillment
|
|
133
|
-
enablement
|
|
134
|
-
self-managed
|
|
135
|
-
saas
|
|
136
|
-
]
|
|
137
|
-
|
|
138
|
-
tests.sort_by do |test|
|
|
139
|
-
ordering.index(test.stage) || ordering.size
|
|
140
|
-
end.group_by(&:stage)
|
|
141
|
-
end
|
|
142
|
-
|
|
143
|
-
def generate_testcase_listing_by_status(tests:, tests_by_status:)
|
|
144
|
-
failed_tests = tests_by_status['failed']
|
|
145
|
-
passed_tests = tests_by_status['passed']
|
|
146
|
-
other_tests = tests.reject do |test|
|
|
147
|
-
test.status == 'failed' || test.status == 'passed'
|
|
148
|
-
end
|
|
149
|
-
|
|
150
|
-
[
|
|
151
|
-
(failed_listings(failed_tests) if failed_tests),
|
|
152
|
-
(passed_listings(passed_tests) if passed_tests),
|
|
153
|
-
(other_listings(other_tests) if other_tests.any?)
|
|
154
|
-
].compact.join("\n\n")
|
|
155
|
-
end
|
|
156
|
-
|
|
157
|
-
def failed_listings(failed_tests)
|
|
158
|
-
generate_testcase_listing(failed_tests)
|
|
159
|
-
end
|
|
160
|
-
|
|
161
|
-
def passed_listings(passed_tests)
|
|
162
|
-
<<~MARKDOWN.chomp
|
|
163
|
-
<details><summary>Passed tests:</summary>
|
|
164
|
-
|
|
165
|
-
#{generate_testcase_listing(passed_tests, passed: true)}
|
|
166
|
-
|
|
167
|
-
</details>
|
|
168
|
-
MARKDOWN
|
|
169
|
-
end
|
|
170
|
-
|
|
171
|
-
def other_listings(other_tests)
|
|
172
|
-
<<~MARKDOWN.chomp
|
|
173
|
-
<details><summary>Other tests:</summary>
|
|
174
|
-
|
|
175
|
-
#{generate_testcase_listing(other_tests)}
|
|
176
|
-
|
|
177
|
-
</details>
|
|
178
|
-
MARKDOWN
|
|
179
|
-
end
|
|
180
|
-
|
|
181
|
-
def generate_testcase_listing(tests, passed: false)
|
|
182
|
-
body = tests.group_by(&:testcase).map do |testcase, tests_with_same_testcase|
|
|
183
|
-
tests_with_same_testcase.sort_by!(&:name)
|
|
184
|
-
[
|
|
185
|
-
generate_test_text(testcase, tests_with_same_testcase, passed),
|
|
186
|
-
generate_test_job(tests_with_same_testcase),
|
|
187
|
-
generate_test_status(tests_with_same_testcase),
|
|
188
|
-
generate_test_actions(tests_with_same_testcase)
|
|
189
|
-
].join(' | ')
|
|
190
|
-
end.join("\n")
|
|
191
|
-
|
|
192
|
-
<<~MARKDOWN.chomp
|
|
193
|
-
| Test | Job | Status | Action |
|
|
194
|
-
| - | - | - | - |
|
|
195
|
-
#{body}
|
|
196
|
-
MARKDOWN
|
|
197
|
-
end
|
|
198
|
-
|
|
199
|
-
def generate_test_text(testcase, tests_with_same_testcase, passed)
|
|
200
|
-
text = tests_with_same_testcase.map(&:name).uniq.join(', ')
|
|
201
|
-
|
|
202
|
-
if testcase && !passed
|
|
203
|
-
"[#{text}](#{testcase})"
|
|
204
|
-
else
|
|
205
|
-
text
|
|
206
|
-
end
|
|
207
|
-
end
|
|
208
|
-
|
|
209
|
-
def generate_test_job(tests_with_same_testcase)
|
|
210
|
-
tests_with_same_testcase.map do |test|
|
|
211
|
-
ci_job_id = test.ci_job_url[/\d+\z/]
|
|
212
|
-
|
|
213
|
-
"[#{ci_job_id}](#{test.ci_job_url})#{' ~"quarantine"' if test.quarantine?}"
|
|
214
|
-
end.uniq.join(', ')
|
|
215
|
-
end
|
|
216
|
-
|
|
217
|
-
def generate_test_status(tests_with_same_testcase)
|
|
218
|
-
tests_with_same_testcase.map(&:status).uniq.map do |status|
|
|
219
|
-
%(~"#{status}")
|
|
220
|
-
end.join(', ')
|
|
221
|
-
end
|
|
222
|
-
|
|
223
|
-
def generate_test_actions(tests_with_same_testcase)
|
|
224
|
-
# All failed tests would be grouped together, meaning that
|
|
225
|
-
# if one failed, all the tests here would be failed too.
|
|
226
|
-
# So this check is safe. Same applies to 'passed'.
|
|
227
|
-
# But all other status might be mixing together,
|
|
228
|
-
# we cannot assume other statuses.
|
|
229
|
-
if tests_with_same_testcase.first.status == 'failed'
|
|
230
|
-
tests_having_failure_issue =
|
|
231
|
-
tests_with_same_testcase.select(&:failure_issue)
|
|
232
|
-
|
|
233
|
-
if tests_having_failure_issue.any?
|
|
234
|
-
items = tests_having_failure_issue.uniq(&:failure_issue).map do |test|
|
|
235
|
-
"<li>[ ] [failure issue](#{test.failure_issue})</li>"
|
|
236
|
-
end.join(' ')
|
|
237
|
-
|
|
238
|
-
"<ul>#{items}</ul>"
|
|
239
|
-
else
|
|
240
|
-
'<ul><li>[ ] failure issue exists or was created</li></ul>'
|
|
241
|
-
end
|
|
242
|
-
else
|
|
243
|
-
'-'
|
|
244
|
-
end
|
|
245
|
-
end
|
|
246
|
-
|
|
247
|
-
def generate_qa_issue_relation
|
|
248
|
-
return unless Runtime::Env.qa_issue_url
|
|
249
|
-
|
|
250
|
-
<<~MARKDOWN.chomp
|
|
251
|
-
## Release QA issue
|
|
252
|
-
|
|
253
|
-
* #{Runtime::Env.qa_issue_url}
|
|
254
|
-
|
|
255
|
-
/relate #{Runtime::Env.qa_issue_url}
|
|
256
|
-
MARKDOWN
|
|
257
|
-
end
|
|
258
|
-
|
|
259
|
-
def generate_link_to_dashboard
|
|
260
|
-
return unless Runtime::Env.qa_run_type
|
|
261
|
-
|
|
262
|
-
<<~MARKDOWN.chomp
|
|
263
|
-
## Link to Grafana dashboard for run-type of #{Runtime::Env.qa_run_type}
|
|
264
|
-
|
|
265
|
-
* https://dashboards.quality.gitlab.net/d/tR_SmBDVk/main-runs?orgId=1&refresh=1m&var-run_type=#{Runtime::Env.qa_run_type}
|
|
266
|
-
MARKDOWN
|
|
267
|
-
end
|
|
268
|
-
|
|
269
|
-
def fetch_pipeline_failed_jobs
|
|
270
|
-
failed_jobs = []
|
|
271
|
-
|
|
272
|
-
ci_project_client = Gitlab.client(
|
|
273
|
-
endpoint: Runtime::Env.ci_api_v4_url,
|
|
274
|
-
private_token: ci_project_token)
|
|
275
|
-
|
|
276
|
-
gitlab.handle_gitlab_client_exceptions do
|
|
277
|
-
failed_jobs = ci_project_client.pipeline_jobs(
|
|
278
|
-
Runtime::Env.ci_project_id,
|
|
279
|
-
Runtime::Env.ci_pipeline_id,
|
|
280
|
-
scope: 'failed')
|
|
281
|
-
end
|
|
282
|
-
|
|
283
|
-
failed_jobs
|
|
284
|
-
end
|
|
285
|
-
end
|
|
286
|
-
end
|
|
287
|
-
end
|
|
288
|
-
end
|
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'http'
|
|
4
|
-
require 'json'
|
|
5
|
-
|
|
6
|
-
module GitlabQuality
|
|
7
|
-
module TestTooling
|
|
8
|
-
module Report
|
|
9
|
-
class TestHealthIssueFinder < ReportAsIssue
|
|
10
|
-
HEALTH_PROBLEM_TYPE_TO_LABEL = {
|
|
11
|
-
'pass-after-retry' => 'test-health:pass-after-retry',
|
|
12
|
-
'slow' => 'test-health:slow',
|
|
13
|
-
'failures' => 'test-health:failures'
|
|
14
|
-
}.freeze
|
|
15
|
-
|
|
16
|
-
def initialize(health_problem_type: [], **kwargs)
|
|
17
|
-
super(**kwargs)
|
|
18
|
-
|
|
19
|
-
@health_problem_type = health_problem_type
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
def found_existing_unhealthy_test_issue?
|
|
23
|
-
issue_url = invoke!
|
|
24
|
-
|
|
25
|
-
!issue_url.nil? && !issue_url.empty?
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
def run!
|
|
29
|
-
existing_issue_found = nil
|
|
30
|
-
|
|
31
|
-
applicable_tests.each do |test|
|
|
32
|
-
issues = find_issues_by_hash(test_hash(test), state: 'opened', labels: search_labels)
|
|
33
|
-
next if issues.empty?
|
|
34
|
-
|
|
35
|
-
existing_issue_found = issues.first.web_url
|
|
36
|
-
puts "Found an existing test health issue of type #{health_problem_type} for test #{test.file}:#{test.line_number}: #{existing_issue_found}."
|
|
37
|
-
break
|
|
38
|
-
end
|
|
39
|
-
|
|
40
|
-
puts "Did not find an existing test health issue of type #{health_problem_type}." unless existing_issue_found
|
|
41
|
-
|
|
42
|
-
existing_issue_found
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
def applicable_tests
|
|
46
|
-
applicable_tests = []
|
|
47
|
-
|
|
48
|
-
TestResults::Builder.new(file_glob: files, token: token, project: project).test_results_per_file do |test_results|
|
|
49
|
-
applicable_tests = test_results.select { |test| test_is_applicable?(test) }
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
applicable_tests
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
private
|
|
56
|
-
|
|
57
|
-
attr_reader :health_problem_type
|
|
58
|
-
|
|
59
|
-
# Be mindful about the number of tests this method would return,
|
|
60
|
-
# as we will make at least one API request per test.
|
|
61
|
-
def test_is_applicable?(test)
|
|
62
|
-
expected_test_status =
|
|
63
|
-
case health_problem_type
|
|
64
|
-
when 'failures'
|
|
65
|
-
'failed'
|
|
66
|
-
else
|
|
67
|
-
'passed'
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
test.status == expected_test_status
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
def search_labels
|
|
74
|
-
['test', HEALTH_PROBLEM_TYPE_TO_LABEL[health_problem_type]]
|
|
75
|
-
end
|
|
76
|
-
end
|
|
77
|
-
end
|
|
78
|
-
end
|
|
79
|
-
end
|