gitlab_quality-test_tooling 1.24.0 → 1.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -1
- data/Gemfile.lock +2 -2
- data/exe/update-screenshot-paths +1 -1
- data/lib/gitlab_quality/test_tooling/report/failed_test_issue.rb +2 -1
- data/lib/gitlab_quality/test_tooling/report/flaky_test_issue.rb +7 -7
- data/lib/gitlab_quality/test_tooling/report/update_screenshot_path.rb +2 -2
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb +44 -0
- data/lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb +44 -0
- data/lib/gitlab_quality/test_tooling/{concerns → test_metrics_exporter}/influxdb_tools.rb +1 -1
- data/lib/gitlab_quality/test_tooling/{test_metric → test_metrics_exporter}/log_test_metrics.rb +6 -6
- data/lib/gitlab_quality/test_tooling/{concerns → test_metrics_exporter}/test_metrics.rb +2 -2
- data/lib/gitlab_quality/test_tooling/version.rb +1 -1
- metadata +7 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6895c7a00b6414c417fdadfe15fc030d316c1a6ecac135d5a3b281748d321926
|
4
|
+
data.tar.gz: d9c8dcf80524df5f22c3cc32439f2abaf5f148d061c46a1b8f74a6c40aaa2a21
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c6305ab3fcdf665926a8a1180ffd1308611e39ef53442d584e1797e8caf0115527bac36376b668ee89473d94f86ee160eeeeeaf2a72dca8348a96b44c375f233
|
7
|
+
data.tar.gz: bf77f8ce83ebebb939925479593cb7c21b9f2f4a8ac564da009605d509ec4e56c4eb68b73fbc156b05567c9c0d5603b7ac2552f77c615b1b19fa2e5e19ad568a
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
3.
|
1
|
+
3.2.3
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
gitlab_quality-test_tooling (1.
|
4
|
+
gitlab_quality-test_tooling (1.26.0)
|
5
5
|
activesupport (>= 6.1, < 7.2)
|
6
6
|
amatch (~> 0.4.1)
|
7
7
|
gitlab (~> 4.19)
|
@@ -331,4 +331,4 @@ DEPENDENCIES
|
|
331
331
|
webmock (= 3.7.0)
|
332
332
|
|
333
333
|
BUNDLED WITH
|
334
|
-
2.5.
|
334
|
+
2.5.4
|
data/exe/update-screenshot-paths
CHANGED
@@ -11,7 +11,7 @@ params = {}
|
|
11
11
|
options = OptionParser.new do |opts|
|
12
12
|
opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
|
13
13
|
|
14
|
-
opts.on('-i', '--input-files INPUT_FILES', String, 'RSpec report files (JSON or JUnit XML)') do |input_files|
|
14
|
+
opts.on('-i', '--input-files INPUT_FILES', String, 'Glob pattern for RSpec report files (JSON or JUnit XML)') do |input_files|
|
15
15
|
params[:input_files] = input_files
|
16
16
|
end
|
17
17
|
|
@@ -33,10 +33,11 @@ module GitlabQuality
|
|
33
33
|
input_files:,
|
34
34
|
base_issue_labels: nil,
|
35
35
|
dry_run: false,
|
36
|
+
related_issues_file: nil,
|
36
37
|
project: nil,
|
37
38
|
max_diff_ratio: DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION,
|
38
39
|
**_kwargs)
|
39
|
-
super(token: token, input_files: input_files, project: project, dry_run: dry_run)
|
40
|
+
super(token: token, input_files: input_files, related_issues_file: related_issues_file, project: project, dry_run: dry_run)
|
40
41
|
|
41
42
|
@base_issue_labels = Set.new(base_issue_labels)
|
42
43
|
@max_diff_ratio = max_diff_ratio.to_f
|
@@ -133,17 +133,17 @@ module GitlabQuality
|
|
133
133
|
# See https://gitlab.com/gitlab-org/quality/engineering-productivity/team/-/blob/main/scripts/unhealthy_test_issues_statistics.rb
|
134
134
|
# to gather these statistics.
|
135
135
|
#
|
136
|
-
#
|
137
|
-
# P90
|
138
|
-
# P95
|
139
|
-
#
|
136
|
+
# x <= P90 => flakiness::4
|
137
|
+
# P90 < x <= P95 => flakiness::3
|
138
|
+
# P95 < x <= P99 => flakiness::2
|
139
|
+
# > P99 => flakiness::1
|
140
140
|
def flakiness_status_labels_quick_action(reports_count)
|
141
141
|
case reports_count
|
142
|
-
when
|
142
|
+
when 399..Float::INFINITY
|
143
143
|
'/label ~"flakiness::1"'
|
144
|
-
when
|
144
|
+
when 37..398
|
145
145
|
'/label ~"flakiness::2"'
|
146
|
-
when
|
146
|
+
when 13..36
|
147
147
|
'/label ~"flakiness::3"'
|
148
148
|
else
|
149
149
|
'/label ~"flakiness::4"'
|
@@ -16,8 +16,8 @@ module GitlabQuality
|
|
16
16
|
|
17
17
|
def invoke!
|
18
18
|
Dir.glob(input_files).each do |input_file|
|
19
|
-
rewrite_screenshot_paths_in_junit_file(input_file)
|
20
|
-
rewrite_screenshot_paths_in_json_file(input_file.
|
19
|
+
rewrite_screenshot_paths_in_junit_file(input_file) if input_file.end_with?('.xml')
|
20
|
+
rewrite_screenshot_paths_in_json_file(input_file) if input_file.end_with?('.json')
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "singleton"
|
4
|
+
|
5
|
+
module GitlabQuality
|
6
|
+
module TestTooling
|
7
|
+
module TestMetricsExporter
|
8
|
+
class << self
|
9
|
+
def configuration
|
10
|
+
Config.instance
|
11
|
+
end
|
12
|
+
|
13
|
+
def configure
|
14
|
+
yield(configuration)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
class Config
|
19
|
+
include Singleton
|
20
|
+
|
21
|
+
attr_accessor :influxdb_url,
|
22
|
+
:influxdb_token,
|
23
|
+
:influxdb_bucket,
|
24
|
+
:run_type
|
25
|
+
|
26
|
+
attr_writer :custom_keys_tags,
|
27
|
+
:custom_keys_fields,
|
28
|
+
:test_metric_file_name
|
29
|
+
|
30
|
+
def custom_keys_tags
|
31
|
+
@custom_keys_tags || []
|
32
|
+
end
|
33
|
+
|
34
|
+
def custom_keys_fields
|
35
|
+
@custom_keys_fields || []
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_metric_file_name
|
39
|
+
@test_metric_file_name || "test_metrics.json"
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module GitlabQuality
|
4
|
+
module TestTooling
|
5
|
+
module TestMetricsExporter
|
6
|
+
class Formatter < RSpec::Core::Formatters::BaseFormatter
|
7
|
+
RSpec::Core::Formatters.register(self, :stop)
|
8
|
+
|
9
|
+
def stop(_notification)
|
10
|
+
setup_test_metrics_exporter
|
11
|
+
|
12
|
+
log_test_metrics.push_test_metrics(
|
13
|
+
custom_keys_tags: config.custom_keys_tags,
|
14
|
+
custom_keys_fields: config.custom_keys_fields
|
15
|
+
)
|
16
|
+
|
17
|
+
log_test_metrics.save_test_metrics(
|
18
|
+
file_name: config.test_metric_file_name,
|
19
|
+
custom_keys_tags: config.custom_keys_tags,
|
20
|
+
custom_keys_fields: config.custom_keys_fields
|
21
|
+
)
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
attr_reader :log_test_metrics
|
27
|
+
|
28
|
+
def config
|
29
|
+
Config.configuration
|
30
|
+
end
|
31
|
+
|
32
|
+
def setup_test_metrics_exporter
|
33
|
+
@log_test_metrics = LogTestMetrics.new(
|
34
|
+
examples: notification.examples,
|
35
|
+
influxdb_url: config.influxdb_url,
|
36
|
+
influxdb_token: config.influxdb_token,
|
37
|
+
influxdb_bucket: config.influxdb_bucket,
|
38
|
+
run_type: config.run_type
|
39
|
+
)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
data/lib/gitlab_quality/test_tooling/{test_metric → test_metrics_exporter}/log_test_metrics.rb
RENAMED
@@ -1,13 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'json'
|
4
|
-
|
5
3
|
module GitlabQuality
|
6
4
|
module TestTooling
|
7
|
-
module
|
5
|
+
module TestMetricsExporter
|
8
6
|
class LogTestMetrics
|
9
|
-
include
|
10
|
-
include
|
7
|
+
include TestMetrics
|
8
|
+
include InfluxdbTools
|
11
9
|
|
12
10
|
CUSTOM_METRICS_KEY = :custom_test_metrics
|
13
11
|
|
@@ -39,7 +37,9 @@ module GitlabQuality
|
|
39
37
|
# @param [Array<String>] custom_keys_tags
|
40
38
|
# @param [Array<String>] custom_keys_fields
|
41
39
|
# @return [nil]
|
42
|
-
def save_test_metrics(file_name
|
40
|
+
def save_test_metrics(file_name:, custom_keys_tags: nil, custom_keys_fields: nil)
|
41
|
+
return Runtime::Logger.warn("No file_name provided, not saving test metrics") if file_name.nil?
|
42
|
+
|
43
43
|
@test_metrics ||= examples.filter_map { |example| parse_test_results(example, custom_keys_tags, custom_keys_fields) }
|
44
44
|
file = "tmp/#{file_name}"
|
45
45
|
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module GitlabQuality
|
4
4
|
module TestTooling
|
5
|
-
module
|
5
|
+
module TestMetricsExporter
|
6
6
|
module TestMetrics
|
7
7
|
# Single common timestamp for all exported example metrics to keep data points consistently grouped
|
8
8
|
#
|
@@ -30,7 +30,7 @@ module GitlabQuality
|
|
30
30
|
job_name: job_name,
|
31
31
|
merge_request: merge_request,
|
32
32
|
run_type: run_type,
|
33
|
-
|
33
|
+
feature_category: example.metadata[:feature_category],
|
34
34
|
product_group: example.metadata[:product_group],
|
35
35
|
exception_class: example.execution_result.exception&.class&.to_s,
|
36
36
|
**custom_metrics(example.metadata, custom_keys)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab_quality-test_tooling
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.26.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- GitLab Quality
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-05-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -438,8 +438,6 @@ files:
|
|
438
438
|
- lefthook.yml
|
439
439
|
- lib/gitlab_quality/test_tooling.rb
|
440
440
|
- lib/gitlab_quality/test_tooling/concerns/find_set_dri.rb
|
441
|
-
- lib/gitlab_quality/test_tooling/concerns/influxdb_tools.rb
|
442
|
-
- lib/gitlab_quality/test_tooling/concerns/test_metrics.rb
|
443
441
|
- lib/gitlab_quality/test_tooling/failed_jobs_table.rb
|
444
442
|
- lib/gitlab_quality/test_tooling/gitlab_client/branches_client.rb
|
445
443
|
- lib/gitlab_quality/test_tooling/gitlab_client/branches_dry_client.rb
|
@@ -497,8 +495,12 @@ files:
|
|
497
495
|
- lib/gitlab_quality/test_tooling/test_meta/processor/meta_processor.rb
|
498
496
|
- lib/gitlab_quality/test_tooling/test_meta/test_meta_updater.rb
|
499
497
|
- lib/gitlab_quality/test_tooling/test_metric/json_test_metric.rb
|
500
|
-
- lib/gitlab_quality/test_tooling/test_metric/log_test_metrics.rb
|
501
498
|
- lib/gitlab_quality/test_tooling/test_metrics/json_test_metric_collection.rb
|
499
|
+
- lib/gitlab_quality/test_tooling/test_metrics_exporter/config.rb
|
500
|
+
- lib/gitlab_quality/test_tooling/test_metrics_exporter/formatter.rb
|
501
|
+
- lib/gitlab_quality/test_tooling/test_metrics_exporter/influxdb_tools.rb
|
502
|
+
- lib/gitlab_quality/test_tooling/test_metrics_exporter/log_test_metrics.rb
|
503
|
+
- lib/gitlab_quality/test_tooling/test_metrics_exporter/test_metrics.rb
|
502
504
|
- lib/gitlab_quality/test_tooling/test_result/base_test_result.rb
|
503
505
|
- lib/gitlab_quality/test_tooling/test_result/j_unit_test_result.rb
|
504
506
|
- lib/gitlab_quality/test_tooling/test_result/json_test_result.rb
|