gitlab-rspec-metrics-exporter 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +25 -0
- data/README.md +77 -0
- data/lib/gitlab/rspec-metrics-exporter.rb +4 -0
- data/lib/gitlab/rspec_metrics_exporter/client.rb +62 -0
- data/lib/gitlab/rspec_metrics_exporter/config.rb +86 -0
- data/lib/gitlab/rspec_metrics_exporter/config_helper.rb +126 -0
- data/lib/gitlab/rspec_metrics_exporter/formatter.rb +66 -0
- data/lib/gitlab/rspec_metrics_exporter/test_metrics.rb +227 -0
- data/lib/gitlab/rspec_metrics_exporter/version.rb +7 -0
- metadata +189 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 1951be0502a4f67569cb0fb9963973a93de7886c4c8bab253baef2fc365b6c61
|
|
4
|
+
data.tar.gz: 03505aadc3e234357cb58e453beda21a128483e3e677adb9422dedd2ae106969
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 410cc4f2c8e8c35e4f2edaa78b5af5417cd1d26e356a9993188903118fbb8713682cd290a8c34e7873b777bad413ae89b55e7155cb027fe9c02b1391d652192f
|
|
7
|
+
data.tar.gz: ea95665fe643e7111662240f2be720cd45d031b9171bff1adc301510054afc61163f01de943bff557c2a753b194824068a591c8fcfb37ed3af2bec49f1a2602c
|
data/LICENSE
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
Copyright (c) GitLab Inc
|
|
2
|
+
|
|
3
|
+
With regard to the GitLab Software:
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
|
22
|
+
|
|
23
|
+
For all third party components incorporated into the GitLab Software, those
|
|
24
|
+
components are licensed under the original license provided by the owner of the
|
|
25
|
+
applicable component.
|
data/README.md
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# gitlab-rspec-metrics-exporter
|
|
2
|
+
|
|
3
|
+
An RSpec formatter that collects test execution data and pushes it to the
|
|
4
|
+
GitLab Quality Observer service.
|
|
5
|
+
|
|
6
|
+
## Installation
|
|
7
|
+
|
|
8
|
+
Add it to your `Gemfile`:
|
|
9
|
+
|
|
10
|
+
```ruby
|
|
11
|
+
group :test do
|
|
12
|
+
gem "gitlab-rspec-metrics-exporter"
|
|
13
|
+
end
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Usage
|
|
17
|
+
|
|
18
|
+
### Configuration helper
|
|
19
|
+
|
|
20
|
+
Preferred way to configure the formatter is to use `ConfigHelper` class:
|
|
21
|
+
|
|
22
|
+
```ruby
|
|
23
|
+
require "gitlab/rspec-metrics-exporter"
|
|
24
|
+
|
|
25
|
+
Gitlab::RSpecMetricsExporter::ConfigHelper.configure!("backend_unit")
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
You may pass a block to further customize the exporter:
|
|
29
|
+
|
|
30
|
+
```ruby
|
|
31
|
+
Gitlab::RSpecMetricsExporter::ConfigHelper.configure!("backend_unit") do |config|
|
|
32
|
+
config.extra_rspec_metadata_keys = [:smoke, :reliable]
|
|
33
|
+
config.skip_record_proc = ->(example) { example.metadata[:do_not_export] }
|
|
34
|
+
end
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Configuration options
|
|
38
|
+
|
|
39
|
+
When the `ConfigHelper.configure!` helper is used, the options marked with an
|
|
40
|
+
**Env var** below are populated automatically from the environment. All
|
|
41
|
+
options can also be set explicitly via yielded `config` object; explicit
|
|
42
|
+
values always take precedence over the corresponding environment variable.
|
|
43
|
+
|
|
44
|
+
| Option | Env var | Description |
|
|
45
|
+
| ---------------------------- | ----------------------------- | ------------------------------------------------------------- |
|
|
46
|
+
| `observer_url` | `GLCI_OBSERVER_URL` | Observer service base URL |
|
|
47
|
+
| `observer_token` | `GLCI_OBSERVER_AUTH_TOKEN` | Auth token sent as `X-Gitlab-Token` |
|
|
48
|
+
| `run_type` | `GLCI_TEST_METRICS_RUN_TYPE` | Suite name (falls back to `CI_JOB_NAME` or `"unknown"`) |
|
|
49
|
+
| `extra_rspec_metadata_keys` | _(none)_ | Additional `example.metadata` keys to include in metrics |
|
|
50
|
+
| `spec_file_path_prefix` | _(none)_ | Prepended to file paths (useful for monorepos) |
|
|
51
|
+
| `skip_record_proc` | _(none)_ | Lambda receiving an example; return `true` to skip exporting |
|
|
52
|
+
| `test_retried_proc` | _(none)_ | Lambda receiving an example; return `true` if it was a retry |
|
|
53
|
+
| `custom_metrics_proc` | _(none)_ | Lambda returning a hash merged into every metric record |
|
|
54
|
+
| `logger` | _(none)_ | Ruby `Logger` instance |
|
|
55
|
+
|
|
56
|
+
The config helper also reads two control-flow variables that gate whether the
|
|
57
|
+
exporter activates at all:
|
|
58
|
+
|
|
59
|
+
| Variable | Purpose |
|
|
60
|
+
| -------------------------- | --------------------------------------------------------- |
|
|
61
|
+
| `CI` | Must be set (any non-empty value) for the helper to run |
|
|
62
|
+
| `GLCI_EXPORT_TEST_METRICS` | Set to `"false"` to disable export. Defaults to `"true"`. |
|
|
63
|
+
|
|
64
|
+
## Development
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
bundle install
|
|
68
|
+
bundle exec rspec
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Release
|
|
72
|
+
|
|
73
|
+
Create merge request with version update in [version.rb](lib/gitlab/rspec_metrics_exporter/version.rb) file
|
|
74
|
+
|
|
75
|
+
## License
|
|
76
|
+
|
|
77
|
+
MIT.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "net/http"
|
|
5
|
+
require "uri"
|
|
6
|
+
|
|
7
|
+
module Gitlab
|
|
8
|
+
module RSpecMetricsExporter
|
|
9
|
+
class Client
|
|
10
|
+
class ResponseError < StandardError
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
TESTS_PATH = "/api/v1/tests"
|
|
14
|
+
# Observer enforces a per-request cap; batch above this size to avoid silent failures.
|
|
15
|
+
MAX_BATCH_SIZE = 10_000
|
|
16
|
+
|
|
17
|
+
def initialize(url:, token:)
|
|
18
|
+
@url = url
|
|
19
|
+
@token = token
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# POST array of test metric records to the observer service.
|
|
23
|
+
# Wraps each batch as { "tests" => [...] } and splits oversized payloads
|
|
24
|
+
# into chunks of at most MAX_BATCH_SIZE records.
|
|
25
|
+
#
|
|
26
|
+
# @param tests [Array<Hash>]
|
|
27
|
+
# @return [Boolean] true when every batch succeeds (or input is empty)
|
|
28
|
+
# @raise [ResponseError] on the first non-2xx batch response
|
|
29
|
+
def post_tests(tests) # rubocop:disable Naming/PredicateMethod
|
|
30
|
+
tests.each_slice(MAX_BATCH_SIZE) do |batch|
|
|
31
|
+
response = post_batch(batch)
|
|
32
|
+
next if (200..299).cover?(response.code.to_i)
|
|
33
|
+
|
|
34
|
+
raise ResponseError, "Observer request failed with status #{response.code}: #{response.body}"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
true
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
attr_reader :url, :token
|
|
43
|
+
|
|
44
|
+
def post_batch(batch)
|
|
45
|
+
uri = endpoint_uri
|
|
46
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
47
|
+
http.use_ssl = (uri.scheme == "https")
|
|
48
|
+
|
|
49
|
+
request = Net::HTTP::Post.new(uri.request_uri)
|
|
50
|
+
request["X-Gitlab-Token"] = token
|
|
51
|
+
request["Content-Type"] = "application/json"
|
|
52
|
+
request.body = { tests: batch }.to_json
|
|
53
|
+
|
|
54
|
+
http.request(request)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def endpoint_uri
|
|
58
|
+
URI.parse("#{url.to_s.chomp('/')}#{TESTS_PATH}")
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "logger"
|
|
4
|
+
require "singleton"
|
|
5
|
+
|
|
6
|
+
module Gitlab
|
|
7
|
+
module RSpecMetricsExporter
|
|
8
|
+
class Config
|
|
9
|
+
include Singleton
|
|
10
|
+
|
|
11
|
+
class << self
|
|
12
|
+
def configuration
|
|
13
|
+
Config.instance
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def configure
|
|
17
|
+
yield(configuration)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
attr_accessor :run_type,
|
|
22
|
+
:observer_url,
|
|
23
|
+
:observer_token
|
|
24
|
+
attr_writer :extra_rspec_metadata_keys,
|
|
25
|
+
:skip_record_proc,
|
|
26
|
+
:test_retried_proc,
|
|
27
|
+
:custom_metrics_proc,
|
|
28
|
+
:spec_file_path_prefix,
|
|
29
|
+
:logger
|
|
30
|
+
|
|
31
|
+
# Whether observer export is configured
|
|
32
|
+
#
|
|
33
|
+
# Export is considered enabled when all required attributes are set
|
|
34
|
+
#
|
|
35
|
+
# @return [Boolean]
|
|
36
|
+
def observer_configured?
|
|
37
|
+
[observer_url, observer_token].none? { |value| value.nil? || value.to_s.empty? }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Extra rspec metadata keys to include in exported metrics
|
|
41
|
+
#
|
|
42
|
+
# @return [Array<Symbol>]
|
|
43
|
+
def extra_rspec_metadata_keys
|
|
44
|
+
@extra_rspec_metadata_keys ||= []
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Extra path prefix for constructing full file path within mono-repository setups
|
|
48
|
+
#
|
|
49
|
+
# @return [String]
|
|
50
|
+
def spec_file_path_prefix
|
|
51
|
+
@spec_file_path_prefix ||= ""
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# A lambda that determines whether to skip recording a test result
|
|
55
|
+
#
|
|
56
|
+
# This is useful when you would want to skip initial failure when retrying specs is set up in a separate process
|
|
57
|
+
# and you want to avoid duplicate records
|
|
58
|
+
#
|
|
59
|
+
# @return [Proc]
|
|
60
|
+
def skip_record_proc
|
|
61
|
+
@skip_record_proc ||= ->(_example) { false }
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# A lambda that determines whether a test was retried or not
|
|
65
|
+
#
|
|
66
|
+
# @return [Proc]
|
|
67
|
+
def test_retried_proc
|
|
68
|
+
@test_retried_proc ||= ->(_example) { false }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# A lambda that return hash with additional custom metrics
|
|
72
|
+
#
|
|
73
|
+
# @return [Proc]
|
|
74
|
+
def custom_metrics_proc
|
|
75
|
+
@custom_metrics_proc ||= ->(_example) { {} }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Logger instance
|
|
79
|
+
#
|
|
80
|
+
# @return [Logger]
|
|
81
|
+
def logger
|
|
82
|
+
@logger ||= Logger.new($stdout, level: Logger::INFO)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "logger"
|
|
4
|
+
|
|
5
|
+
require_relative "config"
|
|
6
|
+
require_relative "formatter"
|
|
7
|
+
|
|
8
|
+
module Gitlab
|
|
9
|
+
module RSpecMetricsExporter
|
|
10
|
+
# Optional convenience helper that wires up the formatter using GitLab CI environment variables.
|
|
11
|
+
#
|
|
12
|
+
# Usage in spec_helper.rb:
|
|
13
|
+
#
|
|
14
|
+
# require "gitlab/rspec_metrics_exporter/config_helper"
|
|
15
|
+
# Gitlab::RSpecMetricsExporter::ConfigHelper.configure!("backend_unit")
|
|
16
|
+
class ConfigHelper
|
|
17
|
+
STABLE_EE_BRANCH_REGEX = /^[\d-]+-stable-ee$/
|
|
18
|
+
|
|
19
|
+
class << self
|
|
20
|
+
def configure!(run_type = nil)
|
|
21
|
+
return unless ENV.fetch("CI", nil) && ENV.fetch("GLCI_EXPORT_TEST_METRICS", "true") == "true"
|
|
22
|
+
|
|
23
|
+
::RSpec.configure do |rspec_config|
|
|
24
|
+
next if rspec_config.dry_run?
|
|
25
|
+
|
|
26
|
+
Config.configure do |exporter_config|
|
|
27
|
+
self.logger = exporter_config.logger
|
|
28
|
+
|
|
29
|
+
yield(exporter_config) if block_given?
|
|
30
|
+
configure_exporter!(exporter_config, run_type)
|
|
31
|
+
next if observer_not_fully_configured?(exporter_config)
|
|
32
|
+
|
|
33
|
+
rspec_config.add_formatter Formatter
|
|
34
|
+
|
|
35
|
+
logger.info("Test metrics export is enabled for run type: #{exporter_config.run_type}")
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
attr_writer :logger
|
|
43
|
+
|
|
44
|
+
def logger
|
|
45
|
+
@logger ||= Logger.new($stdout)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def present?(value)
|
|
49
|
+
!value.nil? && !value.to_s.empty?
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def observer_not_fully_configured?(config)
|
|
53
|
+
if [config.observer_url, ENV.fetch("GLCI_OBSERVER_URL", nil)].none? { |opt| present?(opt) }
|
|
54
|
+
logger.warn("Observer url is not configured!. Set GLCI_OBSERVER_URL environment variable or set observer_url in the exporter config.") # rubocop:disable Layout/LineLength
|
|
55
|
+
return true
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
if [config.observer_token, ENV.fetch("GLCI_OBSERVER_AUTH_TOKEN", nil)].none? { |opt| present?(opt) }
|
|
59
|
+
logger.warn("Observer auth token is not configured!. Set GLCI_OBSERVER_AUTH_TOKEN environment variable or set observer_token in the exporter config.") # rubocop:disable Layout/LineLength
|
|
60
|
+
return true
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
false
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def configure_exporter!(config, run_type)
|
|
67
|
+
config.run_type = run_type || default_run_type unless present?(config.run_type)
|
|
68
|
+
config.custom_metrics_proc = custom_metrics_proc
|
|
69
|
+
|
|
70
|
+
configure_observer!(config)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def configure_observer!(config)
|
|
74
|
+
config.observer_url = observer_url unless present?(config.observer_url)
|
|
75
|
+
config.observer_token = observer_token unless present?(config.observer_token)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def warn_missing_observer_variables
|
|
79
|
+
missing = REQUIRED_OBSERVER_ENV_VARS.reject { |var| present?(ENV.fetch(var, nil)) }
|
|
80
|
+
logger.warn("Test metrics export is enabled but missing environment variables: #{missing.join(', ')}")
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def custom_metrics_proc
|
|
84
|
+
proc do |_example|
|
|
85
|
+
{ pipeline_type: pipeline_type, ci_pipeline_id: ci_pipeline_id }
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def default_branch?
|
|
90
|
+
ENV["CI_COMMIT_REF_NAME"] == ENV["CI_DEFAULT_BRANCH"]
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def pipeline_type
|
|
94
|
+
@pipeline_type ||= detect_pipeline_type
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def detect_pipeline_type
|
|
98
|
+
return "default_branch_scheduled_pipeline" if default_branch? && present?(ENV.fetch("SCHEDULE_TYPE", nil))
|
|
99
|
+
return "default_branch_pipeline" if default_branch?
|
|
100
|
+
return "stable_branch_pipeline" if ENV["CI_COMMIT_REF_NAME"]&.match?(STABLE_EE_BRANCH_REGEX)
|
|
101
|
+
return "backport_merge_request_pipeline" if ENV["CI_MERGE_REQUEST_TARGET_BRANCH_NAME"]&.match?(STABLE_EE_BRANCH_REGEX)
|
|
102
|
+
return "merge_request_pipeline" if present?(ENV["CI_MERGE_REQUEST_IID"])
|
|
103
|
+
return "downstream_pipeline" if ENV["CI_PIPELINE_SOURCE"] == "pipeline"
|
|
104
|
+
|
|
105
|
+
"unknown"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def default_run_type
|
|
109
|
+
@default_run_type ||= ENV.fetch("GLCI_TEST_METRICS_RUN_TYPE") { ENV.fetch("CI_JOB_NAME", nil) }
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def observer_url
|
|
113
|
+
ENV.fetch("GLCI_OBSERVER_URL", nil)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def observer_token
|
|
117
|
+
ENV.fetch("GLCI_OBSERVER_AUTH_TOKEN", nil)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def ci_pipeline_id
|
|
121
|
+
(ENV["PARENT_PIPELINE_ID"] || ENV.fetch("CI_PIPELINE_ID", nil)).to_i
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rspec/core/formatters/base_formatter"
|
|
4
|
+
|
|
5
|
+
require_relative "config"
|
|
6
|
+
require_relative "test_metrics"
|
|
7
|
+
require_relative "client"
|
|
8
|
+
|
|
9
|
+
module Gitlab
|
|
10
|
+
module RSpecMetricsExporter
|
|
11
|
+
class Formatter < RSpec::Core::Formatters::BaseFormatter
|
|
12
|
+
RSpec::Core::Formatters.register(self, :stop)
|
|
13
|
+
|
|
14
|
+
LOG_PREFIX = "[MetricsExporter]"
|
|
15
|
+
|
|
16
|
+
def stop(notification)
|
|
17
|
+
logger.debug("#{LOG_PREFIX} Starting test metrics export")
|
|
18
|
+
data = notification.examples.filter_map do |example|
|
|
19
|
+
next if config.skip_record_proc.call(example)
|
|
20
|
+
|
|
21
|
+
TestMetrics.new(example, time).data
|
|
22
|
+
end
|
|
23
|
+
return logger.warn("#{LOG_PREFIX} No test execution records found, metrics will not be exported!") if data.empty?
|
|
24
|
+
|
|
25
|
+
push_to_observer(data)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def config
|
|
31
|
+
Config.configuration
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def logger
|
|
35
|
+
config.logger
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Single common timestamp for all exported example metrics to keep data points consistently grouped
|
|
39
|
+
#
|
|
40
|
+
# @return [String]
|
|
41
|
+
def time
|
|
42
|
+
return @time if @time
|
|
43
|
+
|
|
44
|
+
ci_created_at = ENV.fetch("CI_PIPELINE_CREATED_AT", nil)
|
|
45
|
+
@time = (ci_created_at ? Time.strptime(ci_created_at, "%Y-%m-%dT%H:%M:%S%z") : Time.now.utc).strftime("%Y-%m-%dT%H:%M:%S.%6N")
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Push data to observer service
|
|
49
|
+
#
|
|
50
|
+
# @param data [Array<Hash>]
|
|
51
|
+
# @return [void]
|
|
52
|
+
def push_to_observer(data)
|
|
53
|
+
return logger.debug("#{LOG_PREFIX} Observer configuration missing, skipping export!") unless config.observer_configured?
|
|
54
|
+
|
|
55
|
+
observer_client.post_tests(data)
|
|
56
|
+
logger.info("#{LOG_PREFIX} Successfully pushed #{data.size} entries to Observer!")
|
|
57
|
+
rescue StandardError => e
|
|
58
|
+
logger.error("#{LOG_PREFIX} Error occurred while pushing metrics to Observer: #{e.message}")
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def observer_client
|
|
62
|
+
Client.new(url: config.observer_url, token: config.observer_token)
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "openssl"
|
|
4
|
+
require "time"
|
|
5
|
+
|
|
6
|
+
require_relative "config"
|
|
7
|
+
|
|
8
|
+
module Gitlab
|
|
9
|
+
module RSpecMetricsExporter
|
|
10
|
+
class TestMetrics
|
|
11
|
+
def initialize(example, timestamp)
|
|
12
|
+
@example = example
|
|
13
|
+
@timestamp = timestamp
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Test data hash
|
|
17
|
+
#
|
|
18
|
+
# @return [Hash]
|
|
19
|
+
def data
|
|
20
|
+
{
|
|
21
|
+
timestamp: timestamp,
|
|
22
|
+
**rspec_metrics,
|
|
23
|
+
**ci_metrics,
|
|
24
|
+
**custom_metrics
|
|
25
|
+
}.compact
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
attr_reader :example, :timestamp
|
|
31
|
+
|
|
32
|
+
# Exporter configuration
|
|
33
|
+
#
|
|
34
|
+
# @return [Config]
|
|
35
|
+
def config
|
|
36
|
+
Config.configuration
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Rspec related metrics
|
|
40
|
+
#
|
|
41
|
+
# @return [Hash]
|
|
42
|
+
def rspec_metrics # rubocop:disable Metrics/AbcSize
|
|
43
|
+
{
|
|
44
|
+
id: without_relative_path(example.id),
|
|
45
|
+
name: example.full_description,
|
|
46
|
+
hash: OpenSSL::Digest.hexdigest("SHA256", "#{file_path}#{example.full_description}")[..40],
|
|
47
|
+
file_path: file_path,
|
|
48
|
+
status: example.execution_result.status,
|
|
49
|
+
run_time: (example.execution_result.run_time * 1000).round,
|
|
50
|
+
location: example_location,
|
|
51
|
+
exception_classes: exception_classes.map { |e| e.class.to_s }.uniq,
|
|
52
|
+
failure_exception: failure_exception,
|
|
53
|
+
quarantined: quarantined?,
|
|
54
|
+
quarantine_issue_url: quarantine_issue_url || "",
|
|
55
|
+
feature_category: example.metadata[:feature_category] || "",
|
|
56
|
+
test_retried: config.test_retried_proc.call(example),
|
|
57
|
+
run_type: run_type,
|
|
58
|
+
spec_file_path_prefix: config.spec_file_path_prefix
|
|
59
|
+
}
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# CI related metrics
|
|
63
|
+
#
|
|
64
|
+
# @return [Hash]
|
|
65
|
+
def ci_metrics
|
|
66
|
+
{
|
|
67
|
+
ci_project_id: env("CI_PROJECT_ID")&.to_i,
|
|
68
|
+
ci_project_path: env("CI_PROJECT_PATH"),
|
|
69
|
+
ci_job_name: ci_job_name,
|
|
70
|
+
ci_job_id: env("CI_JOB_ID")&.to_i,
|
|
71
|
+
ci_pipeline_id: env("CI_PIPELINE_ID")&.to_i,
|
|
72
|
+
ci_merge_request_iid: (env("CI_MERGE_REQUEST_IID") || env("TOP_UPSTREAM_MERGE_REQUEST_IID"))&.to_i,
|
|
73
|
+
ci_branch: env("CI_COMMIT_REF_NAME"),
|
|
74
|
+
ci_target_branch: env("CI_MERGE_REQUEST_TARGET_BRANCH_NAME"),
|
|
75
|
+
ci_server_url: env("CI_SERVER_URL")
|
|
76
|
+
}
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Additional custom metrics
|
|
80
|
+
#
|
|
81
|
+
# @return [Hash]
|
|
82
|
+
def custom_metrics
|
|
83
|
+
metrics = example.metadata
|
|
84
|
+
.slice(*config.extra_rspec_metadata_keys)
|
|
85
|
+
.merge(config.custom_metrics_proc.call(example))
|
|
86
|
+
|
|
87
|
+
metrics.each_with_object({}) do |(k, value), custom_metrics|
|
|
88
|
+
custom_metrics[k.to_sym] = metrics_value(value)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Checks if spec is quarantined
|
|
93
|
+
#
|
|
94
|
+
# @return [String]
|
|
95
|
+
def quarantined?
|
|
96
|
+
return false unless example.metadata.key?(:quarantine)
|
|
97
|
+
|
|
98
|
+
# if quarantine key is present and status is pending, consider it quarantined
|
|
99
|
+
example.execution_result.status == :pending
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Extract quarantine issue URL from metadata
|
|
103
|
+
#
|
|
104
|
+
# @return [String, nil]
|
|
105
|
+
def quarantine_issue_url
|
|
106
|
+
return nil unless example.metadata.key?(:quarantine)
|
|
107
|
+
|
|
108
|
+
metadata = example.metadata[:quarantine]
|
|
109
|
+
case metadata
|
|
110
|
+
when String
|
|
111
|
+
# Direct URL: quarantine: 'https://gitlab.com/.../issues/123'
|
|
112
|
+
metadata if metadata.start_with?("http")
|
|
113
|
+
when Hash
|
|
114
|
+
# Hash format: quarantine: { issue: 'https://...', reason: '...' }
|
|
115
|
+
issue = metadata[:issue] || metadata["issue"]
|
|
116
|
+
# Handle array of URLs (take the first one)
|
|
117
|
+
issue.is_a?(Array) ? issue.first : issue
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Base ci job name
|
|
122
|
+
#
|
|
123
|
+
# @return [String]
|
|
124
|
+
def ci_job_name
|
|
125
|
+
env("CI_JOB_NAME")&.gsub(%r{ \d{1,2}/\d{1,2}}, "")
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Example location
|
|
129
|
+
#
|
|
130
|
+
# @return [String]
|
|
131
|
+
def example_location
|
|
132
|
+
return @example_location if @example_location
|
|
133
|
+
|
|
134
|
+
# ensures that location will be correct even in case of shared examples
|
|
135
|
+
file = example
|
|
136
|
+
.metadata
|
|
137
|
+
.fetch(:shared_group_inclusion_backtrace)
|
|
138
|
+
.last
|
|
139
|
+
&.formatted_inclusion_location
|
|
140
|
+
|
|
141
|
+
return without_relative_path(example.location) unless file
|
|
142
|
+
|
|
143
|
+
@example_location = without_relative_path(file)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# File path based on actual test location, not shared example location
|
|
147
|
+
#
|
|
148
|
+
# @return [String]
|
|
149
|
+
def file_path
|
|
150
|
+
@file_path ||= example_location.gsub(/:\d+$/, "")
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Failure exception classes
|
|
154
|
+
#
|
|
155
|
+
# @return [Array<Exception>]
|
|
156
|
+
def exception_classes
|
|
157
|
+
exception = example.execution_result.exception
|
|
158
|
+
return [] unless exception
|
|
159
|
+
return [exception] unless exception.respond_to?(:all_exceptions)
|
|
160
|
+
|
|
161
|
+
exception.all_exceptions.flatten
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Truncated exception message
|
|
165
|
+
#
|
|
166
|
+
# For MultipleExceptionError, returns the first wrapped exception's message
|
|
167
|
+
# instead of the unhelpful wrapper class name.
|
|
168
|
+
#
|
|
169
|
+
# @return [String]
|
|
170
|
+
def failure_exception
|
|
171
|
+
exception = example.execution_result.exception
|
|
172
|
+
return unless exception
|
|
173
|
+
|
|
174
|
+
source = if exception.respond_to?(:all_exceptions)
|
|
175
|
+
exception.all_exceptions.flatten.first || exception
|
|
176
|
+
else
|
|
177
|
+
exception
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
source.to_s.tr("\n", " ").slice(0, 1000)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# Test run type | suite name
|
|
184
|
+
#
|
|
185
|
+
# @return [String]
|
|
186
|
+
def run_type
|
|
187
|
+
config.run_type || ci_job_name || "unknown"
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# Return non empty environment variable value
|
|
191
|
+
#
|
|
192
|
+
# @param [String] name
|
|
193
|
+
# @return [String, nil]
|
|
194
|
+
def env(name)
|
|
195
|
+
return unless ENV[name] && !ENV[name].empty?
|
|
196
|
+
|
|
197
|
+
ENV.fetch(name)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# Metrics value cast to a valid type
|
|
201
|
+
#
|
|
202
|
+
# @param value [Object]
|
|
203
|
+
# @return [Object]
|
|
204
|
+
def metrics_value(value)
|
|
205
|
+
return value if value.is_a?(Numeric) || value.is_a?(String) || bool?(value) || value.nil?
|
|
206
|
+
|
|
207
|
+
value.to_s
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# Value is a true or false
|
|
211
|
+
#
|
|
212
|
+
# @param val [Object]
|
|
213
|
+
# @return [Boolean]
|
|
214
|
+
def bool?(val)
|
|
215
|
+
[true, false].include?(val)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# Path without leading ./
|
|
219
|
+
#
|
|
220
|
+
# @param path [String]
|
|
221
|
+
# @return [String]
|
|
222
|
+
def without_relative_path(path)
|
|
223
|
+
path.gsub(%r{^\./}, "")
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: gitlab-rspec-metrics-exporter
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.0.1
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Developer Experience
|
|
8
|
+
bindir: bin
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: logger
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - ">="
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '1.4'
|
|
19
|
+
- - "<"
|
|
20
|
+
- !ruby/object:Gem::Version
|
|
21
|
+
version: '2.0'
|
|
22
|
+
type: :runtime
|
|
23
|
+
prerelease: false
|
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
25
|
+
requirements:
|
|
26
|
+
- - ">="
|
|
27
|
+
- !ruby/object:Gem::Version
|
|
28
|
+
version: '1.4'
|
|
29
|
+
- - "<"
|
|
30
|
+
- !ruby/object:Gem::Version
|
|
31
|
+
version: '2.0'
|
|
32
|
+
- !ruby/object:Gem::Dependency
|
|
33
|
+
name: rspec-core
|
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
|
35
|
+
requirements:
|
|
36
|
+
- - ">="
|
|
37
|
+
- !ruby/object:Gem::Version
|
|
38
|
+
version: '3.12'
|
|
39
|
+
- - "<"
|
|
40
|
+
- !ruby/object:Gem::Version
|
|
41
|
+
version: '4.0'
|
|
42
|
+
type: :runtime
|
|
43
|
+
prerelease: false
|
|
44
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
45
|
+
requirements:
|
|
46
|
+
- - ">="
|
|
47
|
+
- !ruby/object:Gem::Version
|
|
48
|
+
version: '3.12'
|
|
49
|
+
- - "<"
|
|
50
|
+
- !ruby/object:Gem::Version
|
|
51
|
+
version: '4.0'
|
|
52
|
+
- !ruby/object:Gem::Dependency
|
|
53
|
+
name: climate_control
|
|
54
|
+
requirement: !ruby/object:Gem::Requirement
|
|
55
|
+
requirements:
|
|
56
|
+
- - "~>"
|
|
57
|
+
- !ruby/object:Gem::Version
|
|
58
|
+
version: '1.2'
|
|
59
|
+
type: :development
|
|
60
|
+
prerelease: false
|
|
61
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
62
|
+
requirements:
|
|
63
|
+
- - "~>"
|
|
64
|
+
- !ruby/object:Gem::Version
|
|
65
|
+
version: '1.2'
|
|
66
|
+
- !ruby/object:Gem::Dependency
|
|
67
|
+
name: debug
|
|
68
|
+
requirement: !ruby/object:Gem::Requirement
|
|
69
|
+
requirements:
|
|
70
|
+
- - "~>"
|
|
71
|
+
- !ruby/object:Gem::Version
|
|
72
|
+
version: '1.11'
|
|
73
|
+
type: :development
|
|
74
|
+
prerelease: false
|
|
75
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
76
|
+
requirements:
|
|
77
|
+
- - "~>"
|
|
78
|
+
- !ruby/object:Gem::Version
|
|
79
|
+
version: '1.11'
|
|
80
|
+
- !ruby/object:Gem::Dependency
|
|
81
|
+
name: rake
|
|
82
|
+
requirement: !ruby/object:Gem::Requirement
|
|
83
|
+
requirements:
|
|
84
|
+
- - "~>"
|
|
85
|
+
- !ruby/object:Gem::Version
|
|
86
|
+
version: '13.0'
|
|
87
|
+
type: :development
|
|
88
|
+
prerelease: false
|
|
89
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
90
|
+
requirements:
|
|
91
|
+
- - "~>"
|
|
92
|
+
- !ruby/object:Gem::Version
|
|
93
|
+
version: '13.0'
|
|
94
|
+
- !ruby/object:Gem::Dependency
|
|
95
|
+
name: rspec
|
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
|
97
|
+
requirements:
|
|
98
|
+
- - "~>"
|
|
99
|
+
- !ruby/object:Gem::Version
|
|
100
|
+
version: '3.12'
|
|
101
|
+
type: :development
|
|
102
|
+
prerelease: false
|
|
103
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
104
|
+
requirements:
|
|
105
|
+
- - "~>"
|
|
106
|
+
- !ruby/object:Gem::Version
|
|
107
|
+
version: '3.12'
|
|
108
|
+
- !ruby/object:Gem::Dependency
|
|
109
|
+
name: rubocop
|
|
110
|
+
requirement: !ruby/object:Gem::Requirement
|
|
111
|
+
requirements:
|
|
112
|
+
- - "~>"
|
|
113
|
+
- !ruby/object:Gem::Version
|
|
114
|
+
version: '1.81'
|
|
115
|
+
type: :development
|
|
116
|
+
prerelease: false
|
|
117
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
118
|
+
requirements:
|
|
119
|
+
- - "~>"
|
|
120
|
+
- !ruby/object:Gem::Version
|
|
121
|
+
version: '1.81'
|
|
122
|
+
- !ruby/object:Gem::Dependency
|
|
123
|
+
name: rubocop-rspec
|
|
124
|
+
requirement: !ruby/object:Gem::Requirement
|
|
125
|
+
requirements:
|
|
126
|
+
- - "~>"
|
|
127
|
+
- !ruby/object:Gem::Version
|
|
128
|
+
version: '3.7'
|
|
129
|
+
type: :development
|
|
130
|
+
prerelease: false
|
|
131
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
132
|
+
requirements:
|
|
133
|
+
- - "~>"
|
|
134
|
+
- !ruby/object:Gem::Version
|
|
135
|
+
version: '3.7'
|
|
136
|
+
- !ruby/object:Gem::Dependency
|
|
137
|
+
name: webmock
|
|
138
|
+
requirement: !ruby/object:Gem::Requirement
|
|
139
|
+
requirements:
|
|
140
|
+
- - "~>"
|
|
141
|
+
- !ruby/object:Gem::Version
|
|
142
|
+
version: '3.19'
|
|
143
|
+
type: :development
|
|
144
|
+
prerelease: false
|
|
145
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
146
|
+
requirements:
|
|
147
|
+
- - "~>"
|
|
148
|
+
- !ruby/object:Gem::Version
|
|
149
|
+
version: '3.19'
|
|
150
|
+
description: An RSpec formatter that collects test execution data and pushes it to
|
|
151
|
+
the GitLab Observer service
|
|
152
|
+
executables: []
|
|
153
|
+
extensions: []
|
|
154
|
+
extra_rdoc_files: []
|
|
155
|
+
files:
|
|
156
|
+
- LICENSE
|
|
157
|
+
- README.md
|
|
158
|
+
- lib/gitlab/rspec-metrics-exporter.rb
|
|
159
|
+
- lib/gitlab/rspec_metrics_exporter/client.rb
|
|
160
|
+
- lib/gitlab/rspec_metrics_exporter/config.rb
|
|
161
|
+
- lib/gitlab/rspec_metrics_exporter/config_helper.rb
|
|
162
|
+
- lib/gitlab/rspec_metrics_exporter/formatter.rb
|
|
163
|
+
- lib/gitlab/rspec_metrics_exporter/test_metrics.rb
|
|
164
|
+
- lib/gitlab/rspec_metrics_exporter/version.rb
|
|
165
|
+
homepage: https://gitlab.com/gitlab-org/quality/analytics/test-metrics-exporters
|
|
166
|
+
licenses:
|
|
167
|
+
- MIT
|
|
168
|
+
metadata:
|
|
169
|
+
homepage_uri: https://gitlab.com/gitlab-org/quality/analytics/test-metrics-exporters
|
|
170
|
+
source_code_uri: https://gitlab.com/gitlab-org/quality/analytics/test-metrics-exporters
|
|
171
|
+
changelog_uri: https://gitlab.com/gitlab-org/quality/analytics/test-metrics-exporters/-/releases
|
|
172
|
+
rdoc_options: []
|
|
173
|
+
require_paths:
|
|
174
|
+
- lib
|
|
175
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
176
|
+
requirements:
|
|
177
|
+
- - ">="
|
|
178
|
+
- !ruby/object:Gem::Version
|
|
179
|
+
version: 3.2.0
|
|
180
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
181
|
+
requirements:
|
|
182
|
+
- - ">="
|
|
183
|
+
- !ruby/object:Gem::Version
|
|
184
|
+
version: '0'
|
|
185
|
+
requirements: []
|
|
186
|
+
rubygems_version: 4.0.6
|
|
187
|
+
specification_version: 4
|
|
188
|
+
summary: RSpec formatter that exports test execution metrics to GitLab Observer.
|
|
189
|
+
test_files: []
|