gitlab_quality-test_tooling 2.16.0 → 2.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -1
- data/.tool-versions +1 -1
- data/Gemfile.lock +30 -28
- data/exe/epic-readiness-notification +58 -0
- data/exe/relate-failure-issue +5 -0
- data/lib/gitlab_quality/test_tooling/click_house/client.rb +111 -0
- data/lib/gitlab_quality/test_tooling/feature_readiness/concerns/issue_concern.rb +1 -1
- data/lib/gitlab_quality/test_tooling/feature_readiness/concerns/work_item_concern.rb +11 -0
- data/lib/gitlab_quality/test_tooling/feature_readiness/epic_readiness_notifier.rb +308 -0
- data/lib/gitlab_quality/test_tooling/gcs_tools.rb +49 -0
- data/lib/gitlab_quality/test_tooling/gitlab_client/gitlab_client.rb +2 -9
- data/lib/gitlab_quality/test_tooling/gitlab_client/group_labels_client.rb +34 -0
- data/lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb +1 -1
- data/lib/gitlab_quality/test_tooling/gitlab_client/issues_dry_client.rb +2 -2
- data/lib/gitlab_quality/test_tooling/report/failed_test_issue.rb +1 -1
- data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb +2 -2
- data/lib/gitlab_quality/test_tooling/report/generate_test_session.rb +2 -2
- data/lib/gitlab_quality/test_tooling/report/group_issues/error_message_normalizer.rb +49 -0
- data/lib/gitlab_quality/test_tooling/report/group_issues/error_pattern_matcher.rb +36 -0
- data/lib/gitlab_quality/test_tooling/report/group_issues/failure_processor.rb +73 -0
- data/lib/gitlab_quality/test_tooling/report/group_issues/group_results_in_issues.rb +48 -0
- data/lib/gitlab_quality/test_tooling/report/group_issues/incident_checker.rb +61 -0
- data/lib/gitlab_quality/test_tooling/report/group_issues/issue_base.rb +48 -0
- data/lib/gitlab_quality/test_tooling/report/group_issues/issue_creator.rb +44 -0
- data/lib/gitlab_quality/test_tooling/report/group_issues/issue_finder.rb +81 -0
- data/lib/gitlab_quality/test_tooling/report/group_issues/issue_formatter.rb +83 -0
- data/lib/gitlab_quality/test_tooling/report/group_issues/issue_manager.rb +33 -0
- data/lib/gitlab_quality/test_tooling/report/group_issues/issue_updater.rb +87 -0
- data/lib/gitlab_quality/test_tooling/report/health_problem_reporter.rb +6 -3
- data/lib/gitlab_quality/test_tooling/report/knapsack_report_issue.rb +1 -1
- data/lib/gitlab_quality/test_tooling/report/merge_request_slow_tests_report.rb +2 -6
- data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb +134 -5
- data/lib/gitlab_quality/test_tooling/report/slow_test_issue.rb +2 -1
- data/lib/gitlab_quality/test_tooling/runtime/env.rb +9 -4
- data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/api_log_finder.rb +1 -1
- data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/application_log_finder.rb +1 -1
- data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/exception_log_finder.rb +1 -1
- data/lib/gitlab_quality/test_tooling/system_logs/finders/rails/graphql_log_finder.rb +1 -1
- data/lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb +39 -11
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb +115 -15
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb +57 -36
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/test_metrics.rb +124 -80
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/utils.rb +94 -0
- data/lib/gitlab_quality/test_tooling/test_result/base_test_result.rb +6 -2
- data/lib/gitlab_quality/test_tooling/version.rb +1 -1
- data/lib/gitlab_quality/test_tooling.rb +3 -0
- metadata +70 -55
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/log_test_metrics.rb +0 -117
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/support/gcs_tools.rb +0 -49
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/support/influxdb_tools.rb +0 -33
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module GitlabQuality
|
|
4
|
+
module TestTooling
|
|
5
|
+
module Report
|
|
6
|
+
module GroupIssues
|
|
7
|
+
class IssueFormatter
|
|
8
|
+
def generate_issue_title(grouped_failure)
|
|
9
|
+
case grouped_failure[:pattern_name]
|
|
10
|
+
when /http_500/ then "Environment Issue: HTTP 500 Internal Server Errors"
|
|
11
|
+
when /http_400/ then "Environment Issue: Backend Connection Failures"
|
|
12
|
+
when /http_503/ then "Environment Issue: Service Unavailable (503)"
|
|
13
|
+
when /timeout/ then "Environment Issue: Timeout Failures"
|
|
14
|
+
when /git_rpc|repository/ then "Environment Issue: Repository/Git Operation Failures"
|
|
15
|
+
else "Environment Issue: Multiple Similar Failures"
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def generate_issue_description(grouped_failure, options = {})
|
|
20
|
+
active_incidents = IncidentChecker.get_active_incidents(token: options[:token])
|
|
21
|
+
incident_section = IncidentChecker.format_incidents_for_issue(active_incidents)
|
|
22
|
+
|
|
23
|
+
<<~MARKDOWN
|
|
24
|
+
## Environment Issue: #{grouped_failure[:pattern_name]}
|
|
25
|
+
|
|
26
|
+
Multiple tests have failed with similar error patterns, indicating an environment-related issue affecting multiple test cases.
|
|
27
|
+
|
|
28
|
+
### Error Pattern
|
|
29
|
+
```
|
|
30
|
+
#{grouped_failure[:normalized_message]}
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
### Affected Tests (#{grouped_failure[:failures].size} failures)
|
|
34
|
+
#{format_affected_tests(grouped_failure[:failures])}
|
|
35
|
+
|
|
36
|
+
### Pipeline Information
|
|
37
|
+
#{format_pipeline_info(grouped_failure[:failures].first)}
|
|
38
|
+
|
|
39
|
+
### Recommended Actions
|
|
40
|
+
#{generate_recommended_actions(grouped_failure)}
|
|
41
|
+
|
|
42
|
+
#{incident_section}
|
|
43
|
+
---
|
|
44
|
+
<!-- grouped-failure-fingerprint:#{grouped_failure[:fingerprint]} -->
|
|
45
|
+
MARKDOWN
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def format_affected_tests(failures)
|
|
49
|
+
failures.map do |failure|
|
|
50
|
+
job_name = failure[:ci_job_url] || failure.dig(:ci_job, :name) || 'unknown_job'
|
|
51
|
+
spec_file = failure[:file_path] || failure[:file] || 'unknown_spec'
|
|
52
|
+
line_number = failure[:line_number] || failure[:line]
|
|
53
|
+
test_name = failure[:description] || failure[:test_name] || 'Unknown test'
|
|
54
|
+
|
|
55
|
+
spec_with_line = line_number.to_s.empty? ? spec_file : "#{spec_file}:#{line_number}"
|
|
56
|
+
"- **#{test_name}** (Job: `#{job_name}`, Spec: `#{spec_with_line}`)"
|
|
57
|
+
end.join("\n")
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def format_pipeline_info(failure)
|
|
61
|
+
pipeline_url = ENV['CI_PIPELINE_URL'] || "Pipeline #{ENV.fetch('CI_PIPELINE_ID', 'unknown')}"
|
|
62
|
+
job_url = failure[:ci_job_url] || 'Unknown job'
|
|
63
|
+
|
|
64
|
+
"- **Pipeline**: #{pipeline_url}\n- **Job**: #{job_url}"
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def generate_recommended_actions(grouped_failure)
|
|
68
|
+
case grouped_failure[:pattern_name]
|
|
69
|
+
when /http_500/
|
|
70
|
+
"1. Check GitLab instance status and logs\n2. Verify database connectivity\n3. Review application server health"
|
|
71
|
+
when /timeout/
|
|
72
|
+
"1. Check network connectivity\n2. Review timeout configurations\n3. Monitor system resources"
|
|
73
|
+
when /git_rpc|repository/
|
|
74
|
+
"1. Verify Git repository accessibility\n2. Check Gitaly service status\n3. Review storage capacity"
|
|
75
|
+
else
|
|
76
|
+
"1. Check if there are ongoing incidents affecting the GitLab instance\n2. Verify API endpoints are responding correctly\n3. Review system logs for related errors"
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module GitlabQuality
|
|
4
|
+
module TestTooling
|
|
5
|
+
module Report
|
|
6
|
+
module GroupIssues
|
|
7
|
+
class IssueManager
|
|
8
|
+
DEFAULT_MAX_AGE_HOURS = 24
|
|
9
|
+
ISSUES_PER_PAGE = 50
|
|
10
|
+
GROUPED_ISSUE_LABELS = Set.new(%w[test failure::test-environment automation:bot-authored type::maintenance]).freeze
|
|
11
|
+
|
|
12
|
+
def initialize(options = {})
|
|
13
|
+
@options = options
|
|
14
|
+
@client = options[:gitlab]
|
|
15
|
+
@issue_finder = IssueFinder.new(@client, @options)
|
|
16
|
+
@issue_updater = IssueUpdater.new(@client, @options)
|
|
17
|
+
@issue_creator = IssueCreator.new(@client, @options)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def create_or_update_issue(grouped_failure)
|
|
21
|
+
existing_issue = @issue_finder.find_existing_issue(grouped_failure)
|
|
22
|
+
|
|
23
|
+
if existing_issue
|
|
24
|
+
@issue_updater.update_existing_issue(existing_issue, grouped_failure)
|
|
25
|
+
else
|
|
26
|
+
@issue_creator.create_new_issue(grouped_failure)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module GitlabQuality
|
|
4
|
+
module TestTooling
|
|
5
|
+
module Report
|
|
6
|
+
module GroupIssues
|
|
7
|
+
class IssueUpdater < IssueBase
|
|
8
|
+
def update_existing_issue(issue, grouped_failure)
|
|
9
|
+
log_issue_update(issue, grouped_failure)
|
|
10
|
+
append_failures_to_issue(issue, grouped_failure[:failures])
|
|
11
|
+
add_update_comment(issue, grouped_failure[:failures].size)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
private
|
|
15
|
+
|
|
16
|
+
def append_failures_to_issue(issue, failures)
|
|
17
|
+
current_issue = @client.find_issues(iid: issue.iid).first
|
|
18
|
+
return unless current_issue
|
|
19
|
+
|
|
20
|
+
existing_description = current_issue.description
|
|
21
|
+
affected_tests_match = existing_description.match(/### Affected Tests \((\d+) failures?\)/)
|
|
22
|
+
return unless affected_tests_match
|
|
23
|
+
|
|
24
|
+
updated_description = build_updated_description(existing_description, affected_tests_match, failures)
|
|
25
|
+
update_issue_description(issue, updated_description, failures.size)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def update_issue_description(issue, updated_description, failure_count)
|
|
29
|
+
handle_gitlab_api_error("updating issue", "##{issue.web_url}") do
|
|
30
|
+
@client.edit_issue(iid: issue.iid, options: { description: updated_description })
|
|
31
|
+
Runtime::Logger.info "Successfully appended #{failure_count} failures to issue #{issue.web_url}"
|
|
32
|
+
true
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def build_updated_description(existing_description, affected_tests_match, failures)
|
|
37
|
+
current_count = affected_tests_match[1].to_i
|
|
38
|
+
new_count = current_count + failures.size
|
|
39
|
+
|
|
40
|
+
updated_description = existing_description.gsub(
|
|
41
|
+
/### Affected Tests \(\d+ failures?\)/,
|
|
42
|
+
"### Affected Tests (#{new_count} failures)"
|
|
43
|
+
)
|
|
44
|
+
insert_new_failures(updated_description, failures)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def insert_new_failures(description, failures)
|
|
48
|
+
formatter = IssueFormatter.new
|
|
49
|
+
new_failure_entries = formatter.format_affected_tests(failures)
|
|
50
|
+
|
|
51
|
+
test_section_end = description.index('### Pipeline Information')
|
|
52
|
+
return description unless test_section_end
|
|
53
|
+
|
|
54
|
+
insertion_point = description.rindex("\n", test_section_end - 1)
|
|
55
|
+
return description unless insertion_point
|
|
56
|
+
|
|
57
|
+
"#{description[0..insertion_point]}#{new_failure_entries}\n#{description[insertion_point + 1..]}"
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def add_update_comment(issue, failure_count)
|
|
61
|
+
pipeline_url = ENV['CI_PIPELINE_URL'] || "Pipeline #{ENV.fetch('CI_PIPELINE_ID', nil)}"
|
|
62
|
+
comment = build_update_comment(pipeline_url, failure_count)
|
|
63
|
+
add_comment_to_issue(issue, comment)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def build_update_comment(pipeline_url, failure_count)
|
|
67
|
+
"🔄 **New failures added from #{pipeline_url}**\n\n" \
|
|
68
|
+
"Added #{failure_count} additional test failures with the same error pattern."
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def add_comment_to_issue(issue, comment)
|
|
72
|
+
handle_gitlab_api_error("adding comment to issue", issue.web_url) do
|
|
73
|
+
@client.create_issue_note(iid: issue.iid, note: comment)
|
|
74
|
+
Runtime::Logger.info "Comment added successfully to issue #{issue.web_url}"
|
|
75
|
+
true
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def log_issue_update(issue, grouped_failure)
|
|
80
|
+
pipeline_id = ENV.fetch('CI_PIPELINE_ID', nil)
|
|
81
|
+
Runtime::Logger.info "Updating existing issue ##{issue.iid} with #{grouped_failure[:failures].size} new failures from pipeline #{pipeline_id}"
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'date'
|
|
4
|
+
require 'time'
|
|
5
|
+
|
|
3
6
|
module GitlabQuality
|
|
4
7
|
module TestTooling
|
|
5
8
|
module Report
|
|
@@ -15,7 +18,6 @@ module GitlabQuality
|
|
|
15
18
|
class HealthProblemReporter < ReportAsIssue
|
|
16
19
|
include Concerns::GroupAndCategoryLabels
|
|
17
20
|
include Concerns::IssueReports
|
|
18
|
-
include TestMetricsExporter::Support::GcsTools
|
|
19
21
|
|
|
20
22
|
BASE_SEARCH_LABELS = ['test'].freeze
|
|
21
23
|
FOUND_IN_MR_LABEL = '~"found:in MR"'
|
|
@@ -151,7 +153,7 @@ module GitlabQuality
|
|
|
151
153
|
def push_test_to_gcs(tests_data, test_results_filename)
|
|
152
154
|
Runtime::Logger.info "will push the test data to GCS"
|
|
153
155
|
|
|
154
|
-
gcs_client(project_id: gcs_project_id, credentials: gcs_credentials, dry_run: dry_run).put_object(
|
|
156
|
+
GcsTools.gcs_client(project_id: gcs_project_id, credentials: gcs_credentials, dry_run: dry_run).put_object(
|
|
155
157
|
gcs_bucket,
|
|
156
158
|
gcs_metrics_file_name(test_results_filename),
|
|
157
159
|
tests_data.to_json,
|
|
@@ -163,7 +165,7 @@ module GitlabQuality
|
|
|
163
165
|
end
|
|
164
166
|
|
|
165
167
|
def gcs_metrics_file_name(test_results_filename)
|
|
166
|
-
today = Time.now.
|
|
168
|
+
today = Time.now.to_date.iso8601
|
|
167
169
|
|
|
168
170
|
"#{today}-#{test_results_filename}"
|
|
169
171
|
end
|
|
@@ -253,6 +255,7 @@ module GitlabQuality
|
|
|
253
255
|
issue_url: issues.first&.web_url,
|
|
254
256
|
job_id: Runtime::Env.ci_job_id,
|
|
255
257
|
job_web_url: test.ci_job_url,
|
|
258
|
+
job_status: Runtime::Env.ci_job_status,
|
|
256
259
|
pipeline_id: Runtime::Env.ci_pipeline_id,
|
|
257
260
|
pipeline_ref: Runtime::Env.ci_commit_ref_name,
|
|
258
261
|
pipeline_web_url: Runtime::Env.ci_pipeline_url,
|
|
@@ -18,7 +18,7 @@ module GitlabQuality
|
|
|
18
18
|
|
|
19
19
|
NEW_ISSUE_LABELS = Set.new([
|
|
20
20
|
'test', 'automation:bot-authored', 'type::maintenance', 'maintenance::performance',
|
|
21
|
-
'priority::3', 'severity::3', 'knapsack_report'
|
|
21
|
+
'priority::3', 'severity::3', 'knapsack_report', 'suppress-contributor-links'
|
|
22
22
|
]).freeze
|
|
23
23
|
SEARCH_LABELS = %w[test maintenance::performance knapsack_report].freeze
|
|
24
24
|
JOB_TIMEOUT_EPIC_URL = 'https://gitlab.com/groups/gitlab-org/quality/engineering-productivity/-/epics/19'
|
|
@@ -78,13 +78,9 @@ module GitlabQuality
|
|
|
78
78
|
end
|
|
79
79
|
|
|
80
80
|
def slow_test_rows(slow_test)
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
slow_test.each do |test|
|
|
84
|
-
rows << slow_test_table_row(test)
|
|
81
|
+
slow_test.map do |test|
|
|
82
|
+
slow_test_table_row(test)
|
|
85
83
|
end
|
|
86
|
-
|
|
87
|
-
rows
|
|
88
84
|
end
|
|
89
85
|
|
|
90
86
|
def build_note(slow_test)
|
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
require 'nokogiri'
|
|
4
4
|
require 'rubygems/text'
|
|
5
5
|
|
|
6
|
+
require_relative 'group_issues/error_pattern_matcher'
|
|
7
|
+
require_relative 'group_issues/error_message_normalizer'
|
|
8
|
+
require_relative 'group_issues/group_results_in_issues'
|
|
9
|
+
|
|
6
10
|
module GitlabQuality
|
|
7
11
|
module TestTooling
|
|
8
12
|
module Report
|
|
@@ -12,6 +16,7 @@ module GitlabQuality
|
|
|
12
16
|
# - Takes a project where failure issues should be created
|
|
13
17
|
# - Find issue by title (with test description or test file), then further filter by stack trace, then pick the better-matching one
|
|
14
18
|
# - Add the failed job to the issue description, and update labels
|
|
19
|
+
# - Can group similar failures together when group_similar option is enabled
|
|
15
20
|
class RelateFailureIssue < ReportAsIssue
|
|
16
21
|
include TestTooling::Concerns::FindSetDri
|
|
17
22
|
include Concerns::GroupAndCategoryLabels
|
|
@@ -23,7 +28,7 @@ module GitlabQuality
|
|
|
23
28
|
FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
|
|
24
29
|
ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)\n*\n###/m
|
|
25
30
|
|
|
26
|
-
NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2 automation:bot-authored type::maintenance]).freeze
|
|
31
|
+
NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2 automation:bot-authored type::maintenance suppress-contributor-links]).freeze
|
|
27
32
|
SCREENSHOT_IGNORED_ERRORS = ['500 Internal Server Error', 'fabricate_via_api!', 'Error Code 500'].freeze
|
|
28
33
|
FAILURE_ISSUE_GUIDE_URL = "https://handbook.gitlab.com/handbook/engineering/testing/guide-to-e2e-test-failure-issues/"
|
|
29
34
|
FAILURE_ISSUE_HANDBOOK_GUIDE = "**:rotating_light: [End-to-End Test Failure Issue Debugging Guide](#{FAILURE_ISSUE_GUIDE_URL}) :rotating_light:**\n".freeze
|
|
@@ -31,13 +36,15 @@ module GitlabQuality
|
|
|
31
36
|
# there before being released to the public repository
|
|
32
37
|
DIFF_PROJECT_MAPPINGS = {
|
|
33
38
|
'gitlab-org/quality/e2e-test-issues' => 'gitlab-org/security/gitlab',
|
|
39
|
+
'gitlab-org/quality/test-failure-issues' => 'gitlab-org/security/gitlab',
|
|
34
40
|
'gitlab-org/gitlab' => 'gitlab-org/security/gitlab',
|
|
35
41
|
'gitlab-org/customers-gitlab-com' => 'gitlab-org/customers-gitlab-com'
|
|
36
42
|
}.freeze
|
|
37
43
|
|
|
38
44
|
# Don't use the E2E test issues project for commit parent
|
|
39
45
|
COMMIT_PROJECT_MAPPINGS = {
|
|
40
|
-
'gitlab-org/quality/e2e-test-issues' => 'gitlab-org/gitlab'
|
|
46
|
+
'gitlab-org/quality/e2e-test-issues' => 'gitlab-org/gitlab',
|
|
47
|
+
'gitlab-org/quality/test-failure-issues' => 'gitlab-org/gitlab'
|
|
41
48
|
}.freeze
|
|
42
49
|
|
|
43
50
|
# The project contains record of the deployments we use to determine the commit diff
|
|
@@ -63,6 +70,7 @@ module GitlabQuality
|
|
|
63
70
|
base_issue_labels: nil,
|
|
64
71
|
exclude_labels_for_search: nil,
|
|
65
72
|
metrics_files: [],
|
|
73
|
+
group_similar: false,
|
|
66
74
|
**kwargs)
|
|
67
75
|
super
|
|
68
76
|
@max_diff_ratio = max_diff_ratio.to_f
|
|
@@ -72,21 +80,112 @@ module GitlabQuality
|
|
|
72
80
|
@issue_type = 'issue'
|
|
73
81
|
@commented_issue_list = Set.new
|
|
74
82
|
@metrics_files = Array(metrics_files)
|
|
83
|
+
@group_similar = group_similar
|
|
75
84
|
end
|
|
76
85
|
|
|
77
86
|
private
|
|
78
87
|
|
|
79
|
-
attr_reader :max_diff_ratio, :system_logs, :base_issue_labels, :exclude_labels_for_search, :metrics_files, :ops_gitlab_client
|
|
88
|
+
attr_reader :max_diff_ratio, :system_logs, :base_issue_labels, :exclude_labels_for_search, :metrics_files, :ops_gitlab_client, :group_similar
|
|
80
89
|
|
|
81
90
|
def run!
|
|
82
91
|
puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
|
83
92
|
|
|
93
|
+
run_with_grouping! if group_similar
|
|
94
|
+
|
|
95
|
+
return if similar_issues_grouped?
|
|
96
|
+
|
|
84
97
|
TestResults::Builder.new(token: token, project: project, file_glob: files).test_results_per_file do |test_results|
|
|
85
98
|
puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
|
|
86
99
|
process_test_results(test_results)
|
|
87
100
|
end
|
|
88
101
|
end
|
|
89
102
|
|
|
103
|
+
def similar_issues_grouped?
|
|
104
|
+
grouper.summary[:grouped_issues].positive?
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def grouper
|
|
108
|
+
@grouper ||= GitlabQuality::TestTooling::Report::GroupIssues::GroupResultsInIssues.new(
|
|
109
|
+
gitlab: gitlab,
|
|
110
|
+
config: grouper_config
|
|
111
|
+
)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def run_with_grouping!
|
|
115
|
+
Runtime::Logger.info "=> Grouping similar failures where possible"
|
|
116
|
+
|
|
117
|
+
all_test_results = collect_all_test_results
|
|
118
|
+
return if all_test_results.empty?
|
|
119
|
+
|
|
120
|
+
Runtime::Logger.info "=> Processing #{all_test_results.count} failures with GroupResultsInIssues"
|
|
121
|
+
|
|
122
|
+
failure_data = convert_test_results_to_failure_data(all_test_results)
|
|
123
|
+
|
|
124
|
+
grouper.process_failures(failure_data)
|
|
125
|
+
grouper.process_issues
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def collect_all_test_results
|
|
129
|
+
all_test_results = []
|
|
130
|
+
|
|
131
|
+
TestResults::Builder.new(token: token, project: project, file_glob: files).test_results_per_file do |test_results|
|
|
132
|
+
Runtime::Logger.info "=> Collecting #{test_results.count} tests from #{test_results.path}"
|
|
133
|
+
all_test_results.concat(test_results.select(&:failures?))
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
all_test_results
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def convert_test_results_to_failure_data(test_results)
|
|
140
|
+
test_results.map do |test|
|
|
141
|
+
{
|
|
142
|
+
description: test.name,
|
|
143
|
+
full_description: test.name,
|
|
144
|
+
file_path: test.relative_file,
|
|
145
|
+
line_number: extract_line_number(test),
|
|
146
|
+
exception: extract_exception_data(test),
|
|
147
|
+
ci_job_url: test.ci_job_url,
|
|
148
|
+
testcase: extract_test_id_or_name(test),
|
|
149
|
+
product_group: extract_product_group(test),
|
|
150
|
+
level: extract_level(test)
|
|
151
|
+
}
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def extract_line_number(test)
|
|
156
|
+
test.respond_to?(:line_number) ? test.line_number : nil
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def extract_exception_data(test)
|
|
160
|
+
{
|
|
161
|
+
'message' => test.failures.first&.dig('message') || test.full_stacktrace || 'Unknown error'
|
|
162
|
+
}
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def extract_product_group(test)
|
|
166
|
+
test.respond_to?(:product_group) ? test.product_group : nil
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def extract_level(test)
|
|
170
|
+
test.respond_to?(:level) ? test.level : nil
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def grouper_config
|
|
174
|
+
{
|
|
175
|
+
thresholds: {
|
|
176
|
+
min_failures_to_group: 2
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def extract_test_id_or_name(test)
|
|
182
|
+
return test.example_id if test.respond_to?(:example_id)
|
|
183
|
+
return test.id if test.respond_to?(:id)
|
|
184
|
+
return test.name if test.respond_to?(:name)
|
|
185
|
+
|
|
186
|
+
"#{test.relative_file}:#{test.respond_to?(:line_number) ? test.line_number : 'unknown'}"
|
|
187
|
+
end
|
|
188
|
+
|
|
90
189
|
def new_issue_labels(test)
|
|
91
190
|
up_to_date_labels(test: test, new_labels: NEW_ISSUE_LABELS + group_and_category_labels_for_test(test))
|
|
92
191
|
end
|
|
@@ -469,10 +568,40 @@ module GitlabQuality
|
|
|
469
568
|
end
|
|
470
569
|
|
|
471
570
|
def new_issue_assignee_id(test)
|
|
472
|
-
|
|
571
|
+
assignee_id = try_feature_category_assignment(test)
|
|
572
|
+
return assignee_id if assignee_id
|
|
573
|
+
|
|
574
|
+
try_product_group_assignment(test)
|
|
575
|
+
end
|
|
576
|
+
|
|
577
|
+
def try_feature_category_assignment(test)
|
|
578
|
+
unless test.respond_to?(:feature_category) && test.feature_category?
|
|
579
|
+
Runtime::Logger.info("No feature_category found for DRI assignment")
|
|
580
|
+
return
|
|
581
|
+
end
|
|
582
|
+
|
|
583
|
+
labels_inference = GitlabQuality::TestTooling::LabelsInference.new
|
|
584
|
+
product_group = labels_inference.product_group_from_feature_category(test.feature_category)
|
|
585
|
+
|
|
586
|
+
unless product_group
|
|
587
|
+
Runtime::Logger.warn("Could not map feature_category '#{test.feature_category}' to product_group")
|
|
588
|
+
return
|
|
589
|
+
end
|
|
590
|
+
|
|
591
|
+
dri = test_dri(product_group, test.stage, test.section)
|
|
592
|
+
Runtime::Logger.info("Assigning #{dri} as DRI for the issue (via feature_category).")
|
|
593
|
+
|
|
594
|
+
gitlab.find_user_id(username: dri)
|
|
595
|
+
end
|
|
596
|
+
|
|
597
|
+
def try_product_group_assignment(test)
|
|
598
|
+
unless test.respond_to?(:product_group) && test.product_group?
|
|
599
|
+
Runtime::Logger.info("No product_group found for DRI assignment")
|
|
600
|
+
return
|
|
601
|
+
end
|
|
473
602
|
|
|
474
603
|
dri = test_dri(test.product_group, test.stage, test.section)
|
|
475
|
-
|
|
604
|
+
Runtime::Logger.info("Assigning #{dri} as DRI for the issue (via product_group).")
|
|
476
605
|
|
|
477
606
|
gitlab.find_user_id(username: dri)
|
|
478
607
|
end
|
|
@@ -11,7 +11,8 @@ module GitlabQuality
|
|
|
11
11
|
# - Add test metadata, duration to the issue with group and category labels
|
|
12
12
|
class SlowTestIssue < HealthProblemReporter
|
|
13
13
|
IDENTITY_LABELS = ['test', 'rspec:slow test', 'test-health:slow', 'rspec profiling', 'automation:bot-authored'].freeze
|
|
14
|
-
NEW_ISSUE_LABELS = Set.new(
|
|
14
|
+
NEW_ISSUE_LABELS = Set.new(
|
|
15
|
+
['test', 'type::maintenance', 'maintenance::performance', 'priority::3', 'severity::3', 'suppress-contributor-links', *IDENTITY_LABELS]).freeze
|
|
15
16
|
REPORT_SECTION_HEADER = '### Slowness reports'
|
|
16
17
|
REPORTS_DOCUMENTATION = <<~DOC
|
|
17
18
|
Slow tests were detected, please see the [test speed best practices guide](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-speed)
|
|
@@ -15,16 +15,17 @@ module GitlabQuality
|
|
|
15
15
|
'CI_JOB_ID' => :ci_job_id,
|
|
16
16
|
'CI_JOB_NAME' => :ci_job_name,
|
|
17
17
|
'CI_JOB_URL' => :ci_job_url,
|
|
18
|
+
'CI_JOB_STATUS' => :ci_job_status,
|
|
18
19
|
'CI_PIPELINE_ID' => :ci_pipeline_id,
|
|
19
20
|
'CI_PIPELINE_NAME' => :ci_pipeline_name,
|
|
20
21
|
'CI_PIPELINE_URL' => :ci_pipeline_url,
|
|
21
22
|
'CI_PROJECT_ID' => :ci_project_id,
|
|
22
23
|
'CI_PROJECT_NAME' => :ci_project_name,
|
|
23
24
|
'CI_PROJECT_PATH' => :ci_project_path,
|
|
25
|
+
'CI_PIPELINE_CREATED_AT' => :ci_pipeline_created_at,
|
|
24
26
|
'DEPLOY_VERSION' => :deploy_version,
|
|
25
27
|
'GITLAB_QA_ISSUE_URL' => :qa_issue_url,
|
|
26
|
-
'QA_GITLAB_CI_TOKEN' => :gitlab_ci_token
|
|
27
|
-
'SLACK_QA_CHANNEL' => :slack_qa_channel
|
|
28
|
+
'QA_GITLAB_CI_TOKEN' => :gitlab_ci_token
|
|
28
29
|
}.freeze
|
|
29
30
|
|
|
30
31
|
ENV_VARIABLES.each do |env_name, method_name|
|
|
@@ -61,6 +62,10 @@ module GitlabQuality
|
|
|
61
62
|
env_var_value_if_defined('GITLAB_GRAPHQL_API_BASE')
|
|
62
63
|
end
|
|
63
64
|
|
|
65
|
+
def slack_alerts_channel
|
|
66
|
+
env_var_value_if_defined('SLACK_ALERTS_CHANNEL') || 'C09HQ5BN07J' # test-tooling-alerts channel ID
|
|
67
|
+
end
|
|
68
|
+
|
|
64
69
|
def pipeline_from_project_name
|
|
65
70
|
%w[gitlab gitaly].any? { |str| ci_project_name.to_s.start_with?(str) } ? default_branch : ci_project_name
|
|
66
71
|
end
|
|
@@ -111,12 +116,12 @@ module GitlabQuality
|
|
|
111
116
|
end
|
|
112
117
|
|
|
113
118
|
def env_var_value_if_defined(variable)
|
|
114
|
-
|
|
119
|
+
ENV.fetch(variable) if env_var_value_valid?(variable)
|
|
115
120
|
end
|
|
116
121
|
|
|
117
122
|
def env_var_name_if_defined(variable)
|
|
118
123
|
# Pass through the variables if they are defined and not empty in the environment
|
|
119
|
-
|
|
124
|
+
"$#{variable}" if env_var_value_valid?(variable)
|
|
120
125
|
end
|
|
121
126
|
end
|
|
122
127
|
end
|
|
@@ -8,9 +8,7 @@ module GitlabQuality
|
|
|
8
8
|
class TestMetaUpdater
|
|
9
9
|
include TestTooling::Concerns::FindSetDri
|
|
10
10
|
|
|
11
|
-
attr_reader :project, :ref, :report_issue, :processed_commits
|
|
12
|
-
|
|
13
|
-
TEST_PLATFORM_MAINTAINERS_SLACK_CHANNEL_ID = 'C0437FV9KBN' # test-platform-maintainers
|
|
11
|
+
attr_reader :project, :ref, :report_issue, :processed_commits, :token, :specs_file, :dry_run, :processor
|
|
14
12
|
|
|
15
13
|
def initialize(token:, project:, specs_file:, processor:, ref: 'master', dry_run: false)
|
|
16
14
|
@specs_file = specs_file
|
|
@@ -269,12 +267,15 @@ module GitlabQuality
|
|
|
269
267
|
# Fetch the id for the dri of the product group and stage
|
|
270
268
|
# The first item returned is the id of the assignee and the second item is the handle
|
|
271
269
|
#
|
|
272
|
-
# @param [
|
|
270
|
+
# @param [Hash] test object
|
|
273
271
|
# @param [String] devops_stage
|
|
272
|
+
# @param [String] section
|
|
274
273
|
# @return [Array<Integer, String>]
|
|
275
|
-
def fetch_dri_id(
|
|
276
|
-
|
|
274
|
+
def fetch_dri_id(test, devops_stage, section)
|
|
275
|
+
product_group = determine_product_group(test)
|
|
276
|
+
return unless product_group
|
|
277
277
|
|
|
278
|
+
assignee_handle = ENV.fetch('QA_TEST_DRI_HANDLE', nil) || test_dri(product_group, devops_stage, section)
|
|
278
279
|
[user_id_for_username(assignee_handle), assignee_handle]
|
|
279
280
|
end
|
|
280
281
|
|
|
@@ -291,7 +292,7 @@ module GitlabQuality
|
|
|
291
292
|
# @param [String] message the message to post
|
|
292
293
|
# @return [HTTP::Response]
|
|
293
294
|
def post_message_on_slack(message)
|
|
294
|
-
channel =
|
|
295
|
+
channel = Runtime::Env.slack_alerts_channel
|
|
295
296
|
slack_options = {
|
|
296
297
|
slack_webhook_url: ENV.fetch('CI_SLACK_WEBHOOK_URL', nil),
|
|
297
298
|
channel: channel,
|
|
@@ -335,6 +336,16 @@ module GitlabQuality
|
|
|
335
336
|
label ? %(/label ~"#{label}") : ''
|
|
336
337
|
end
|
|
337
338
|
|
|
339
|
+
# Infers the group label from the provided feature category
|
|
340
|
+
#
|
|
341
|
+
# @param [String] feature_category feature category
|
|
342
|
+
# @return [String]
|
|
343
|
+
def label_from_feature_category(feature_category)
|
|
344
|
+
labels = labels_inference.infer_labels_from_feature_category(feature_category)
|
|
345
|
+
group_label = labels.find { |label| label.start_with?('group::') }
|
|
346
|
+
group_label ? %(/label ~"#{group_label}") : ''
|
|
347
|
+
end
|
|
348
|
+
|
|
338
349
|
# Returns the link to the Grafana dashboard for single spec metrics
|
|
339
350
|
#
|
|
340
351
|
# @param [String] example_name the full example name
|
|
@@ -344,10 +355,6 @@ module GitlabQuality
|
|
|
344
355
|
base_url + CGI.escape(example_name)
|
|
345
356
|
end
|
|
346
357
|
|
|
347
|
-
private
|
|
348
|
-
|
|
349
|
-
attr_reader :token, :specs_file, :dry_run, :processor
|
|
350
|
-
|
|
351
358
|
# Returns any test description string within single or double quotes
|
|
352
359
|
#
|
|
353
360
|
# @param [String] line the line to check for any quoted string
|
|
@@ -380,6 +387,27 @@ module GitlabQuality
|
|
|
380
387
|
def labels_inference
|
|
381
388
|
@labels_inference ||= GitlabQuality::TestTooling::LabelsInference.new
|
|
382
389
|
end
|
|
390
|
+
|
|
391
|
+
private
|
|
392
|
+
|
|
393
|
+
def determine_product_group(test)
|
|
394
|
+
return map_feature_category_to_product_group(test) if has_feature_category?(test)
|
|
395
|
+
return test.product_group if has_product_group?(test)
|
|
396
|
+
|
|
397
|
+
nil
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
def has_feature_category?(test)
|
|
401
|
+
test.respond_to?(:feature_category) && test.feature_category?
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
def has_product_group?(test)
|
|
405
|
+
test.respond_to?(:product_group) && test.product_group?
|
|
406
|
+
end
|
|
407
|
+
|
|
408
|
+
def map_feature_category_to_product_group(test)
|
|
409
|
+
labels_inference.product_group_from_feature_category(test.feature_category)
|
|
410
|
+
end
|
|
383
411
|
end
|
|
384
412
|
end
|
|
385
413
|
end
|