gitlab_quality-test_tooling 3.16.0 → 3.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENTS.md +7 -4
- data/Gemfile.lock +1 -1
- data/README.md +11 -0
- data/exe/relate-failure-issue +10 -0
- data/exe/test-coverage +91 -63
- data/lib/gitlab_quality/test_tooling/code_coverage/click_house/per_test_coverage_table.rb +41 -8
- data/lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregation.sql +33 -16
- data/lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_score_snapshot.sql +31 -0
- data/lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_score_snapshotter.rb +95 -0
- data/lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data.rb +16 -0
- data/lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_exporter.rb +181 -0
- data/lib/gitlab_quality/test_tooling/gitlab_client/issues_client.rb +24 -0
- data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb +44 -1
- data/lib/gitlab_quality/test_tooling/version.rb +1 -1
- metadata +5 -7
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/client.rb +0 -50
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb +0 -88
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/config_helper.rb +0 -115
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb +0 -67
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/test_metrics.rb +0 -228
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'logger'
|
|
5
|
+
|
|
6
|
+
module GitlabQuality
|
|
7
|
+
module TestTooling
|
|
8
|
+
module CodeCoverage
|
|
9
|
+
# Standalone per-test coverage export: inserts per-test, per-source-file
|
|
10
|
+
# rows into `code_coverage.test_coverage_per_file` and, unless skipped,
|
|
11
|
+
# runs the daily `test_health_risk` aggregation.
|
|
12
|
+
#
|
|
13
|
+
# Unlike the full coverage-metrics export, this path needs only ClickHouse
|
|
14
|
+
# credentials and the per-test coverage glob, not the LCOV report, the
|
|
15
|
+
# Crystalball test map, or the responsibility patterns. Category enrichment
|
|
16
|
+
# (feature_category/group/stage/section per row) is optional: pass
|
|
17
|
+
# `test_reports` to map test files to feature categories via the test
|
|
18
|
+
# report JSON, otherwise those columns stay blank.
|
|
19
|
+
#
|
|
20
|
+
# `skip_aggregation` lets a batched caller insert without re-running the
|
|
21
|
+
# table-wide aggregation on every batch. The streaming export invokes this
|
|
22
|
+
# once per artifact batch with `skip_aggregation: true`, then runs the
|
|
23
|
+
# aggregation once at the end.
|
|
24
|
+
class PerTestCoverageExporter
|
|
25
|
+
# @param clickhouse [Hash] connection params (:url, :database, :username, :password)
|
|
26
|
+
def initialize(
|
|
27
|
+
coverage_glob:, clickhouse:,
|
|
28
|
+
test_reports: nil, jest_quarantine_file: nil,
|
|
29
|
+
captured_sha: ENV.fetch('CI_COMMIT_SHA', ''), skip_aggregation: false, logger: nil)
|
|
30
|
+
@coverage_glob = coverage_glob
|
|
31
|
+
@clickhouse = clickhouse
|
|
32
|
+
@test_reports = test_reports
|
|
33
|
+
@jest_quarantine_file = jest_quarantine_file
|
|
34
|
+
@captured_sha = captured_sha.to_s
|
|
35
|
+
@skip_aggregation = skip_aggregation
|
|
36
|
+
# Default to INFO. The export logs every batched INSERT at debug via
|
|
37
|
+
# the ClickHouse client, and at full-bucket scale that floods the CI
|
|
38
|
+
# job log past its size limit and buries real errors. Callers can pass
|
|
39
|
+
# a debug logger explicitly when they need the SQL.
|
|
40
|
+
@logger = logger || ::Logger.new($stdout, level: ::Logger::INFO)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# @return [void]
|
|
44
|
+
def run
|
|
45
|
+
coverage_files = Dir.glob(coverage_glob)
|
|
46
|
+
if coverage_files.empty?
|
|
47
|
+
logger.info(
|
|
48
|
+
"#{ClickHouse::LOG_PREFIX} No per-test coverage artifacts matched #{coverage_glob}; nothing to export."
|
|
49
|
+
)
|
|
50
|
+
return
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
insert(coverage_files)
|
|
54
|
+
jest_quarantine_error = ingest_jest_quarantine_best_effort
|
|
55
|
+
unless skip_aggregation
|
|
56
|
+
aggregate
|
|
57
|
+
snapshot_scores
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Re-raise only after aggregation has run, so a jest quarantine ingest
|
|
61
|
+
# failure can't silently stop the table-wide rollup while still
|
|
62
|
+
# failing the job loudly for follow-up.
|
|
63
|
+
raise jest_quarantine_error if jest_quarantine_error
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
attr_reader :coverage_glob, :clickhouse, :test_reports, :jest_quarantine_file,
|
|
69
|
+
:captured_sha, :skip_aggregation, :logger
|
|
70
|
+
|
|
71
|
+
def insert(coverage_files)
|
|
72
|
+
tests_to_categories, feature_categories_to_teams = resolve_categories
|
|
73
|
+
|
|
74
|
+
data = PerTestCoverageData.new(
|
|
75
|
+
coverage_files,
|
|
76
|
+
tests_to_categories: tests_to_categories,
|
|
77
|
+
feature_categories_to_teams: feature_categories_to_teams,
|
|
78
|
+
captured_sha: captured_sha
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
ClickHouse::PerTestCoverageTable.new(**clickhouse_credentials).push(data.as_db_table)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def ingest_jest_quarantine
|
|
85
|
+
return unless jest_quarantine_file
|
|
86
|
+
|
|
87
|
+
unless File.exist?(jest_quarantine_file)
|
|
88
|
+
logger.info(
|
|
89
|
+
"#{ClickHouse::LOG_PREFIX} Jest quarantine file not found at #{jest_quarantine_file}; " \
|
|
90
|
+
"skipping jest quarantine ingestion."
|
|
91
|
+
)
|
|
92
|
+
return
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
ClickHouse::JestQuarantinedTestsTable.new(**clickhouse_credentials)
|
|
96
|
+
.populate(quarantine_file: jest_quarantine_file)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# The jest quarantine snapshot is best-effort relative to the
|
|
100
|
+
# aggregation: a failure here (a transient ClickHouse error, say)
|
|
101
|
+
# shouldn't stop the rollup from running on the rows already inserted.
|
|
102
|
+
# Capture the error so #run can aggregate first, then re-raise it.
|
|
103
|
+
def ingest_jest_quarantine_best_effort
|
|
104
|
+
ingest_jest_quarantine
|
|
105
|
+
nil
|
|
106
|
+
rescue StandardError => e
|
|
107
|
+
logger.error(
|
|
108
|
+
"#{ClickHouse::LOG_PREFIX} Jest quarantine ingestion failed; continuing so the " \
|
|
109
|
+
"aggregation still runs: #{e.message}"
|
|
110
|
+
)
|
|
111
|
+
e
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def aggregate
|
|
115
|
+
ClickHouse::TestHealthRiskAggregator.new(**clickhouse_credentials).run
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Freezes the day's per-pillar scores into test_health_score_history,
|
|
119
|
+
# right after the aggregation the score views read. Best-effort: the
|
|
120
|
+
# snapshot is a non-critical add-on, so a failure (for example before
|
|
121
|
+
# the score views exist) must not fail the daily job once the
|
|
122
|
+
# aggregation has succeeded. A missed freeze is recoverable because the
|
|
123
|
+
# dashboard reads the live view for today.
|
|
124
|
+
def snapshot_scores
|
|
125
|
+
ClickHouse::TestHealthScoreSnapshotter.new(**clickhouse_credentials).run
|
|
126
|
+
rescue StandardError => e
|
|
127
|
+
logger.error(
|
|
128
|
+
"#{ClickHouse::LOG_PREFIX} Score snapshot failed; the aggregation still " \
|
|
129
|
+
"succeeded, continuing: #{e.message}"
|
|
130
|
+
)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# `tests_to_categories` comes from the test report JSON; without it, rows
|
|
134
|
+
# carry blank categories. The category-to-team map is a ClickHouse read,
|
|
135
|
+
# so only build both when there are reports to enrich.
|
|
136
|
+
def resolve_categories
|
|
137
|
+
return [{}, {}] unless test_reports
|
|
138
|
+
|
|
139
|
+
# Comma-separated patterns, mirroring Artifacts#test_reports.
|
|
140
|
+
patterns = test_reports.split(',').map(&:strip).reject(&:empty?)
|
|
141
|
+
report_files = Dir.glob(patterns)
|
|
142
|
+
return [{}, {}] if report_files.empty?
|
|
143
|
+
|
|
144
|
+
[tests_to_categories_from(report_files), feature_categories_to_teams]
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Category to group/stage/section, read from `code_coverage.category_owners`
|
|
148
|
+
# (populated by sync-category-owners). This is the source the full
|
|
149
|
+
# coverage export already uses, and it retains categories the live
|
|
150
|
+
# stages.yml fetch has since dropped (global_search and the Duo/AI groups),
|
|
151
|
+
# so more rows resolve to a real group instead of blank. `owner_records`
|
|
152
|
+
# gives string-keyed values; PerTestCoverageData reads symbol keys.
|
|
153
|
+
def feature_categories_to_teams
|
|
154
|
+
ClickHouse::CategoryOwnersTable.new(**clickhouse_credentials).owner_records.transform_values do |owner|
|
|
155
|
+
{ group: owner["group"], stage: owner["stage"], section: owner["section"] }
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def tests_to_categories_from(report_files)
|
|
160
|
+
report_files.each_with_object({}) do |file, combined|
|
|
161
|
+
# Category enrichment is optional, so a malformed or vanished report
|
|
162
|
+
# shouldn't abort the per-test insert (the rows are the point). Warn
|
|
163
|
+
# and skip the file, leaving those tests with blank categories.
|
|
164
|
+
begin
|
|
165
|
+
report = TestReport.new(JSON.parse(File.read(file)))
|
|
166
|
+
rescue JSON::ParserError, Errno::ENOENT => e
|
|
167
|
+
logger.warn("#{ClickHouse::LOG_PREFIX} Skipping unreadable test report #{file}: #{e.message}")
|
|
168
|
+
next
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
combined.merge!(report.tests_to_categories) { |_, old_val, new_val| (old_val + new_val).uniq }
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def clickhouse_credentials
|
|
176
|
+
clickhouse.merge(logger: logger)
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
@@ -43,6 +43,8 @@ module GitlabQuality
|
|
|
43
43
|
# The GitLab client is used for API access: https://github.com/NARKOZ/gitlab
|
|
44
44
|
class IssuesClient < GitlabClient
|
|
45
45
|
REPORTER_ACCESS_LEVEL = 20
|
|
46
|
+
DUO_WORKFLOW_TIMEOUT = 30
|
|
47
|
+
DUO_WORKFLOW_DEFINITION = 'developer/v1'
|
|
46
48
|
|
|
47
49
|
def assert_user_permission!
|
|
48
50
|
handle_gitlab_client_exceptions do
|
|
@@ -123,6 +125,28 @@ module GitlabQuality
|
|
|
123
125
|
end
|
|
124
126
|
end
|
|
125
127
|
|
|
128
|
+
# Starts a GitLab Duo Agent Platform workflow (Flows API).
|
|
129
|
+
# The agent investigates and posts its own analysis back, so no follow-up call is needed.
|
|
130
|
+
#
|
|
131
|
+
# This is a best-effort helper: it deliberately does NOT use handle_gitlab_client_exceptions
|
|
132
|
+
# (which would retry with long backoffs and post failures to Slack) and uses a short timeout
|
|
133
|
+
# so a slow or unavailable Duo endpoint can never stall the reporting job. The caller is
|
|
134
|
+
# expected to rescue any error.
|
|
135
|
+
#
|
|
136
|
+
# project_id accepts either a numeric ID or a namespace path (e.g. 'gitlab-org/quality/...'),
|
|
137
|
+
# matching the GitLab API's project-identifier convention; the path form is verified working.
|
|
138
|
+
def start_duo_workflow(goal:, workflow_definition: DUO_WORKFLOW_DEFINITION, project_id: project)
|
|
139
|
+
client.post('/ai/duo_workflows/workflows',
|
|
140
|
+
timeout: DUO_WORKFLOW_TIMEOUT,
|
|
141
|
+
headers: { 'Content-Type' => 'application/json' },
|
|
142
|
+
body: {
|
|
143
|
+
project_id: project_id,
|
|
144
|
+
goal: goal,
|
|
145
|
+
workflow_definition: workflow_definition,
|
|
146
|
+
start_workflow: true
|
|
147
|
+
}.to_json)
|
|
148
|
+
end
|
|
149
|
+
|
|
126
150
|
def edit_issue_note(issue_iid:, note_id:, note:)
|
|
127
151
|
handle_gitlab_client_exceptions do
|
|
128
152
|
client.edit_issue_note(project, issue_iid, note_id, note)
|
|
@@ -83,12 +83,13 @@ module GitlabQuality
|
|
|
83
83
|
@metrics_files = Array(metrics_files)
|
|
84
84
|
@group_similar = group_similar
|
|
85
85
|
@environment_issues_output_file = environment_issues_output_file
|
|
86
|
+
@duo_analysis_modes = Array(kwargs[:duo_analysis])
|
|
86
87
|
end
|
|
87
88
|
|
|
88
89
|
private
|
|
89
90
|
|
|
90
91
|
attr_reader :max_diff_ratio, :system_logs, :base_issue_labels, :exclude_labels_for_search, :metrics_files, :ops_gitlab_client, :group_similar,
|
|
91
|
-
:environment_issues_output_file
|
|
92
|
+
:environment_issues_output_file, :duo_analysis_modes
|
|
92
93
|
|
|
93
94
|
def run!
|
|
94
95
|
puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
|
@@ -294,9 +295,47 @@ module GitlabQuality
|
|
|
294
295
|
# On a dry run, created_issue may not be populated
|
|
295
296
|
test.failure_issue ||= created_issue&.web_url
|
|
296
297
|
|
|
298
|
+
trigger_duo_analysis(created_issue, test) if duo_analysis_modes.include?('create')
|
|
299
|
+
|
|
297
300
|
created_issue
|
|
298
301
|
end
|
|
299
302
|
|
|
303
|
+
# Best-effort: start a GitLab Duo workflow to investigate the issue; the agent comments its
|
|
304
|
+
# analysis back. A failure here must never block issue creation or the pipeline.
|
|
305
|
+
def trigger_duo_analysis(issue, test)
|
|
306
|
+
return unless issue&.web_url
|
|
307
|
+
|
|
308
|
+
gitlab.start_duo_workflow(goal: duo_analysis_goal(issue, test))
|
|
309
|
+
puts " => Triggered Duo failure analysis workflow for #{issue.web_url}"
|
|
310
|
+
rescue StandardError => e
|
|
311
|
+
warn(" => Could not trigger Duo failure analysis for #{issue.web_url} (ignored): #{e.class}: #{e.message}")
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
def duo_analysis_goal(issue, test)
|
|
315
|
+
<<~GOAL.chomp
|
|
316
|
+
Investigate the newly-reported end-to-end test failure described in this issue: #{issue.web_url}
|
|
317
|
+
(failing spec: `#{test.name}`). Read the issue for the failing spec and stack trace.
|
|
318
|
+
|
|
319
|
+
Investigate the `gitlab-org/gitlab` project (where the product code and E2E specs live —
|
|
320
|
+
note this is NOT the project hosting this issue): inspect its recent commits and merge
|
|
321
|
+
requests on the default branch using GitLab tools, and consult the Orbit knowledge graph
|
|
322
|
+
via MCP tools if it is available. Using those, identify the most likely cause of this
|
|
323
|
+
failure. Then post a comment on that issue with your analysis.
|
|
324
|
+
|
|
325
|
+
The failure may be caused by a product code change, or it may be environmental — for
|
|
326
|
+
example a flaky test, an infrastructure/environment issue, or a CI runner problem. Decide
|
|
327
|
+
which is more likely:
|
|
328
|
+
|
|
329
|
+
- If a specific change or merge request looks responsible, name the likely author in your
|
|
330
|
+
comment and ask them to confirm. Frame this as advisory ("this looks likely related —
|
|
331
|
+
please confirm"), never as an assertion of blame.
|
|
332
|
+
- If the failure looks environmental, flaky, or otherwise not attributable to a specific
|
|
333
|
+
change, explain why — but do not ping or attribute it to anyone.
|
|
334
|
+
|
|
335
|
+
This is an automated, best-effort request to speed up triage of newly-detected failures.
|
|
336
|
+
GOAL
|
|
337
|
+
end
|
|
338
|
+
|
|
300
339
|
def pipeline_issues_with_similar_stacktrace(test)
|
|
301
340
|
search_labels = (base_issue_labels + Set.new(%w[test failure::new])).to_a
|
|
302
341
|
not_labels = exclude_labels_for_search.to_a
|
|
@@ -428,6 +467,10 @@ module GitlabQuality
|
|
|
428
467
|
end
|
|
429
468
|
|
|
430
469
|
def generate_diff_link
|
|
470
|
+
# Outside CI (e.g. local runs) there is no pipeline to diff against, so skip gracefully
|
|
471
|
+
# rather than crashing on a nil pipeline URL or an unsupported project.
|
|
472
|
+
return "No commit diff available (no CI pipeline context)." unless Runtime::Env.ci_pipeline_url
|
|
473
|
+
|
|
431
474
|
initialize_gitlab_ops_client
|
|
432
475
|
|
|
433
476
|
if Runtime::Env.ci_pipeline_url.include?('ops.gitlab.net')
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gitlab_quality-test_tooling
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 3.
|
|
4
|
+
version: 3.19.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- GitLab Quality
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-07-02 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: climate_control
|
|
@@ -499,9 +499,12 @@ files:
|
|
|
499
499
|
- lib/gitlab_quality/test_tooling/code_coverage/click_house/test_file_mappings_table.rb
|
|
500
500
|
- lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregation.sql
|
|
501
501
|
- lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_risk_aggregator.rb
|
|
502
|
+
- lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_score_snapshot.sql
|
|
503
|
+
- lib/gitlab_quality/test_tooling/code_coverage/click_house/test_health_score_snapshotter.rb
|
|
502
504
|
- lib/gitlab_quality/test_tooling/code_coverage/coverage_data.rb
|
|
503
505
|
- lib/gitlab_quality/test_tooling/code_coverage/lcov_file.rb
|
|
504
506
|
- lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_data.rb
|
|
507
|
+
- lib/gitlab_quality/test_tooling/code_coverage/per_test_coverage_exporter.rb
|
|
505
508
|
- lib/gitlab_quality/test_tooling/code_coverage/responsibility_classifier.rb
|
|
506
509
|
- lib/gitlab_quality/test_tooling/code_coverage/responsibility_patterns_config.rb
|
|
507
510
|
- lib/gitlab_quality/test_tooling/code_coverage/rspec_report.rb
|
|
@@ -594,11 +597,6 @@ files:
|
|
|
594
597
|
- lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb
|
|
595
598
|
- lib/gitlab_quality/test_tooling/test_metric/json_test_metric.rb
|
|
596
599
|
- lib/gitlab_quality/test_tooling/test_metrics/json_test_metric_collection.rb
|
|
597
|
-
- lib/gitlab_quality/test_tooling/test_metrics_exporter/client.rb
|
|
598
|
-
- lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb
|
|
599
|
-
- lib/gitlab_quality/test_tooling/test_metrics_exporter/config_helper.rb
|
|
600
|
-
- lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb
|
|
601
|
-
- lib/gitlab_quality/test_tooling/test_metrics_exporter/test_metrics.rb
|
|
602
600
|
- lib/gitlab_quality/test_tooling/test_quarantine/quarantine_formatter.rb
|
|
603
601
|
- lib/gitlab_quality/test_tooling/test_quarantine/quarantine_helper.rb
|
|
604
602
|
- lib/gitlab_quality/test_tooling/test_result/base_test_result.rb
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require "httparty"
|
|
4
|
-
require "json"
|
|
5
|
-
|
|
6
|
-
module GitlabQuality
|
|
7
|
-
module TestTooling
|
|
8
|
-
module TestMetricsExporter
|
|
9
|
-
class Client
|
|
10
|
-
ResponseError = Class.new(StandardError)
|
|
11
|
-
|
|
12
|
-
TESTS_PATH = "/api/v1/tests"
|
|
13
|
-
# Observer enforces a per-request cap; batch above this size to avoid silent failures.
|
|
14
|
-
MAX_BATCH_SIZE = 10_000
|
|
15
|
-
|
|
16
|
-
def initialize(url:, token:)
|
|
17
|
-
@url = url
|
|
18
|
-
@token = token
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
# POST array of test metric records to the observer service.
|
|
22
|
-
# Wraps each batch as { "tests" => [...] } and splits oversized payloads
|
|
23
|
-
# into chunks of at most MAX_BATCH_SIZE records.
|
|
24
|
-
#
|
|
25
|
-
# @param tests [Array<Hash>]
|
|
26
|
-
# @return [Boolean] true when every batch succeeds (or input is empty)
|
|
27
|
-
# @raise [ResponseError] on the first non-2xx batch response
|
|
28
|
-
def post_tests(tests)
|
|
29
|
-
tests.each_slice(MAX_BATCH_SIZE) do |batch|
|
|
30
|
-
response = HTTParty.post(
|
|
31
|
-
"#{url.to_s.chomp('/')}#{TESTS_PATH}",
|
|
32
|
-
body: { tests: batch }.to_json,
|
|
33
|
-
headers: {
|
|
34
|
-
"X-Gitlab-Token" => token,
|
|
35
|
-
"Content-Type" => "application/json"
|
|
36
|
-
}
|
|
37
|
-
)
|
|
38
|
-
raise ResponseError, "Observer request failed with status #{response.code}: #{response.body}" unless response.success?
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
true
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
private
|
|
45
|
-
|
|
46
|
-
attr_reader :url, :token
|
|
47
|
-
end
|
|
48
|
-
end
|
|
49
|
-
end
|
|
50
|
-
end
|
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require "logger"
|
|
4
|
-
require "singleton"
|
|
5
|
-
|
|
6
|
-
module GitlabQuality
|
|
7
|
-
module TestTooling
|
|
8
|
-
module TestMetricsExporter
|
|
9
|
-
class Config
|
|
10
|
-
include Singleton
|
|
11
|
-
|
|
12
|
-
class << self
|
|
13
|
-
def configuration
|
|
14
|
-
Config.instance
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
def configure
|
|
18
|
-
yield(configuration)
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
attr_accessor :run_type,
|
|
23
|
-
:observer_url,
|
|
24
|
-
:observer_token
|
|
25
|
-
attr_writer :extra_rspec_metadata_keys,
|
|
26
|
-
:skip_record_proc,
|
|
27
|
-
:test_retried_proc,
|
|
28
|
-
:custom_metrics_proc,
|
|
29
|
-
:spec_file_path_prefix,
|
|
30
|
-
:logger
|
|
31
|
-
|
|
32
|
-
# Whether observer export is configured
|
|
33
|
-
#
|
|
34
|
-
# Export is considered enabled when all required attributes are set
|
|
35
|
-
#
|
|
36
|
-
# @return [Boolean]
|
|
37
|
-
def observer_configured?
|
|
38
|
-
[observer_url, observer_token].none? { |value| value.nil? || value.to_s.empty? }
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
# Extra rspec metadata keys to include in exported metrics
|
|
42
|
-
#
|
|
43
|
-
# @return [Array<Symbol>]
|
|
44
|
-
def extra_rspec_metadata_keys
|
|
45
|
-
@extra_rspec_metadata_keys ||= []
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
# Extra path prefix for constructing full file path within mono-repository setups
|
|
49
|
-
#
|
|
50
|
-
# @return [String]
|
|
51
|
-
def spec_file_path_prefix
|
|
52
|
-
@spec_file_path_prefix ||= ""
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
# A lambda that determines whether to skip recording a test result
|
|
56
|
-
#
|
|
57
|
-
# This is useful when you would want to skip initial failure when retrying specs is set up in a separate process
|
|
58
|
-
# and you want to avoid duplicate records
|
|
59
|
-
#
|
|
60
|
-
# @return [Proc]
|
|
61
|
-
def skip_record_proc
|
|
62
|
-
@skip_record_proc ||= ->(_example) { false }
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
# A lambda that determines whether a test was retried or not
|
|
66
|
-
#
|
|
67
|
-
# @return [Proc]
|
|
68
|
-
def test_retried_proc
|
|
69
|
-
@test_retried_proc ||= ->(_example) { false }
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
# A lambda that return hash with additional custom metrics
|
|
73
|
-
#
|
|
74
|
-
# @return [Proc]
|
|
75
|
-
def custom_metrics_proc
|
|
76
|
-
@custom_metrics_proc ||= ->(_example) { {} }
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
# Logger instance
|
|
80
|
-
#
|
|
81
|
-
# @return [Logger]
|
|
82
|
-
def logger
|
|
83
|
-
@logger ||= Logger.new($stdout, level: Logger::INFO)
|
|
84
|
-
end
|
|
85
|
-
end
|
|
86
|
-
end
|
|
87
|
-
end
|
|
88
|
-
end
|
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'logger'
|
|
4
|
-
require 'active_support/core_ext/object/blank'
|
|
5
|
-
|
|
6
|
-
require_relative 'config'
|
|
7
|
-
require_relative 'formatter'
|
|
8
|
-
|
|
9
|
-
module GitlabQuality
|
|
10
|
-
module TestTooling
|
|
11
|
-
module TestMetricsExporter
|
|
12
|
-
class ConfigHelper
|
|
13
|
-
REQUIRED_OBSERVER_ENV_VARS = %w[
|
|
14
|
-
GLCI_OBSERVER_URL
|
|
15
|
-
GLCI_OBSERVER_AUTH_TOKEN
|
|
16
|
-
].freeze
|
|
17
|
-
|
|
18
|
-
class << self
|
|
19
|
-
def configure!(run_type = test_run_type)
|
|
20
|
-
return unless ENV.fetch("CI", nil) && ENV.fetch("GLCI_EXPORT_TEST_METRICS", "true") == "true" && run_type
|
|
21
|
-
|
|
22
|
-
RSpec.configure do |rspec_config|
|
|
23
|
-
next if rspec_config.dry_run?
|
|
24
|
-
|
|
25
|
-
Config.configure do |exporter_config|
|
|
26
|
-
self.logger = exporter_config.logger
|
|
27
|
-
next warn_missing_observer_variables unless observer_env_vars_present?
|
|
28
|
-
|
|
29
|
-
yield(exporter_config) if block_given?
|
|
30
|
-
configure_exporter!(exporter_config, run_type)
|
|
31
|
-
|
|
32
|
-
rspec_config.add_formatter Formatter
|
|
33
|
-
|
|
34
|
-
logger.info("Test metrics export is enabled for run type: #{run_type}")
|
|
35
|
-
end
|
|
36
|
-
end
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
private
|
|
40
|
-
|
|
41
|
-
attr_writer :logger
|
|
42
|
-
|
|
43
|
-
def logger
|
|
44
|
-
@logger ||= Logger.new($stdout)
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
def observer_env_vars_present?
|
|
48
|
-
REQUIRED_OBSERVER_ENV_VARS.all? { |var| ENV.fetch(var, nil) && !ENV[var].empty? }
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
def configure_exporter!(config, run_type)
|
|
52
|
-
config.run_type = run_type
|
|
53
|
-
config.custom_metrics_proc = custom_metrics_proc
|
|
54
|
-
|
|
55
|
-
configure_observer!(config)
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
def configure_observer!(config)
|
|
59
|
-
config.observer_url = observer_url
|
|
60
|
-
config.observer_token = observer_token
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
def warn_missing_observer_variables
|
|
64
|
-
missing = REQUIRED_OBSERVER_ENV_VARS.reject { |var| ENV.fetch(var, nil) && !ENV[var].empty? }
|
|
65
|
-
logger.warn("Test metrics export is enabled but missing environment variables: #{missing.join(', ')}")
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
def custom_metrics_proc
|
|
69
|
-
proc do |_example|
|
|
70
|
-
{ pipeline_type: pipeline_type, ci_pipeline_id: ci_pipeline_id }
|
|
71
|
-
end
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
def default_branch?
|
|
75
|
-
ENV["CI_COMMIT_REF_NAME"] == ENV["CI_DEFAULT_BRANCH"]
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
def pipeline_type
|
|
79
|
-
@pipeline_type ||= if default_branch? && ENV["SCHEDULE_TYPE"].present?
|
|
80
|
-
"default_branch_scheduled_pipeline"
|
|
81
|
-
elsif default_branch?
|
|
82
|
-
"default_branch_pipeline"
|
|
83
|
-
elsif ENV["CI_COMMIT_REF_NAME"]&.match?(/^[\d-]+-stable-ee$/)
|
|
84
|
-
"stable_branch_pipeline"
|
|
85
|
-
elsif ENV["CI_MERGE_REQUEST_TARGET_BRANCH_NAME"]&.match?(/^[\d-]+-stable-ee$/)
|
|
86
|
-
"backport_merge_request_pipeline"
|
|
87
|
-
elsif ENV["CI_MERGE_REQUEST_IID"].present?
|
|
88
|
-
"merge_request_pipeline"
|
|
89
|
-
elsif ENV["CI_PIPELINE_SOURCE"] == "pipeline"
|
|
90
|
-
"downstream_pipeline"
|
|
91
|
-
else
|
|
92
|
-
"unknown"
|
|
93
|
-
end
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
def test_run_type
|
|
97
|
-
@run_type ||= ENV.fetch("GLCI_TEST_METRICS_RUN_TYPE", nil)
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
def observer_url
|
|
101
|
-
ENV.fetch("GLCI_OBSERVER_URL", nil)
|
|
102
|
-
end
|
|
103
|
-
|
|
104
|
-
def observer_token
|
|
105
|
-
ENV.fetch("GLCI_OBSERVER_AUTH_TOKEN", nil)
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
def ci_pipeline_id
|
|
109
|
-
(ENV["PARENT_PIPELINE_ID"] || ENV.fetch("CI_PIPELINE_ID", nil)).to_i
|
|
110
|
-
end
|
|
111
|
-
end
|
|
112
|
-
end
|
|
113
|
-
end
|
|
114
|
-
end
|
|
115
|
-
end
|
|
@@ -1,67 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require "rspec/core/formatters/base_formatter"
|
|
4
|
-
|
|
5
|
-
require_relative "test_metrics"
|
|
6
|
-
require_relative "client"
|
|
7
|
-
|
|
8
|
-
module GitlabQuality
|
|
9
|
-
module TestTooling
|
|
10
|
-
module TestMetricsExporter
|
|
11
|
-
class Formatter < RSpec::Core::Formatters::BaseFormatter
|
|
12
|
-
RSpec::Core::Formatters.register(self, :stop)
|
|
13
|
-
|
|
14
|
-
LOG_PREFIX = "[MetricsExporter]"
|
|
15
|
-
|
|
16
|
-
def stop(notification)
|
|
17
|
-
logger.debug("#{LOG_PREFIX} Starting test metrics export")
|
|
18
|
-
data = notification.examples.filter_map do |example|
|
|
19
|
-
next if config.skip_record_proc.call(example)
|
|
20
|
-
|
|
21
|
-
TestMetrics.new(example, time).data
|
|
22
|
-
end
|
|
23
|
-
return logger.warn("#{LOG_PREFIX} No test execution records found, metrics will not be exported!") if data.empty?
|
|
24
|
-
|
|
25
|
-
push_to_observer(data)
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
private
|
|
29
|
-
|
|
30
|
-
def config
|
|
31
|
-
Config.configuration
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def logger
|
|
35
|
-
config.logger
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
# Single common timestamp for all exported example metrics to keep data points consistently grouped
|
|
39
|
-
#
|
|
40
|
-
# @return [String]
|
|
41
|
-
def time
|
|
42
|
-
return @time if @time
|
|
43
|
-
|
|
44
|
-
ci_created_at = ENV.fetch("CI_PIPELINE_CREATED_AT", nil)
|
|
45
|
-
@time = (ci_created_at ? Time.strptime(ci_created_at, '%Y-%m-%dT%H:%M:%S%z') : Time.now.utc).strftime('%Y-%m-%dT%H:%M:%S.%6N')
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
# Push data to observer service
|
|
49
|
-
#
|
|
50
|
-
# @param data [Array<Hash>]
|
|
51
|
-
# @return [void]
|
|
52
|
-
def push_to_observer(data)
|
|
53
|
-
return logger.debug("#{LOG_PREFIX} Observer configuration missing, skipping export!") unless config.observer_configured?
|
|
54
|
-
|
|
55
|
-
observer_client.post_tests(data)
|
|
56
|
-
logger.info("#{LOG_PREFIX} Successfully pushed #{data.size} entries to Observer!")
|
|
57
|
-
rescue StandardError => e
|
|
58
|
-
logger.error("#{LOG_PREFIX} Error occurred while pushing metrics to Observer: #{e.message}")
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
def observer_client
|
|
62
|
-
Client.new(url: config.observer_url, token: config.observer_token)
|
|
63
|
-
end
|
|
64
|
-
end
|
|
65
|
-
end
|
|
66
|
-
end
|
|
67
|
-
end
|