datadog-ci 1.41.0 → 1.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +43 -2
- data/README.md +45 -0
- data/lib/datadog/ci/async_writer.rb +1 -4
- data/lib/datadog/ci/concurrent_span.rb +20 -0
- data/lib/datadog/ci/configuration/components.rb +2 -1
- data/lib/datadog/ci/configuration/extensions.rb +2 -0
- data/lib/datadog/ci/configuration/reconfiguration.rb +29 -0
- data/lib/datadog/ci/configuration/settings.rb +9 -2
- data/lib/datadog/ci/contrib/cucumber/formatter.rb +27 -8
- data/lib/datadog/ci/contrib/cucumber/instrumentation.rb +11 -3
- data/lib/datadog/ci/contrib/cuprite/driver.rb +5 -5
- data/lib/datadog/ci/contrib/knapsack/runner.rb +1 -1
- data/lib/datadog/ci/contrib/minitest/helpers.rb +59 -0
- data/lib/datadog/ci/contrib/minitest/reporter.rb +2 -2
- data/lib/datadog/ci/contrib/minitest/runner.rb +14 -1
- data/lib/datadog/ci/contrib/minitest/test.rb +4 -1
- data/lib/datadog/ci/contrib/rspec/example.rb +12 -2
- data/lib/datadog/ci/contrib/rspec/example_group.rb +3 -2
- data/lib/datadog/ci/contrib/rspec/runner.rb +2 -2
- data/lib/datadog/ci/contrib/selenium/navigation.rb +5 -5
- data/lib/datadog/ci/ext/metadata.rb +168 -0
- data/lib/datadog/ci/ext/settings.rb +1 -0
- data/lib/datadog/ci/ext/test.rb +11 -0
- data/lib/datadog/ci/impacted_tests_detection/component.rb +1 -1
- data/lib/datadog/ci/readonly_test_module.rb +16 -0
- data/lib/datadog/ci/readonly_test_session.rb +16 -0
- data/lib/datadog/ci/remote/library_settings_client.rb +1 -1
- data/lib/datadog/ci/remote/slow_test_retries.rb +3 -2
- data/lib/datadog/ci/span.rb +46 -10
- data/lib/datadog/ci/test.rb +75 -18
- data/lib/datadog/ci/test_discovery/component.rb +7 -1
- data/lib/datadog/ci/test_discovery/null_component.rb +4 -0
- data/lib/datadog/ci/test_impact_analysis/component.rb +21 -10
- data/lib/datadog/ci/test_impact_analysis/skippable.rb +1 -1
- data/lib/datadog/ci/test_management/component.rb +4 -4
- data/lib/datadog/ci/test_management/tests_properties.rb +1 -1
- data/lib/datadog/ci/test_retries/component.rb +34 -18
- data/lib/datadog/ci/test_retries/driver/base.rb +2 -2
- data/lib/datadog/ci/test_retries/driver/retry_flake_detection.rb +12 -4
- data/lib/datadog/ci/test_retries/strategy/retry_flake_detection.rb +2 -2
- data/lib/datadog/ci/test_retries/strategy/retry_flaky_fixed.rb +6 -3
- data/lib/datadog/ci/test_suite.rb +11 -0
- data/lib/datadog/ci/test_tracing/component.rb +28 -7
- data/lib/datadog/ci/test_tracing/context.rb +24 -7
- data/lib/datadog/ci/test_tracing/deprecated_total_coverage_metric.rb +1 -1
- data/lib/datadog/ci/test_tracing/known_tests.rb +1 -1
- data/lib/datadog/ci/test_tracing/null_component.rb +10 -1
- data/lib/datadog/ci/test_tracing/transport.rb +3 -22
- data/lib/datadog/ci/transport/adapters/net.rb +24 -4
- data/lib/datadog/ci/transport/adapters/net_http_client.rb +31 -0
- data/lib/datadog/ci/transport/api/evp_proxy.rb +2 -1
- data/lib/datadog/ci/transport/http.rb +20 -4
- data/lib/datadog/ci/utils/protected_tags.rb +27 -0
- data/lib/datadog/ci/utils/rum.rb +1 -1
- data/lib/datadog/ci/version.rb +1 -1
- data/lib/datadog/ci.rb +3 -0
- metadata +4 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: dc0b93e52d2bdb75bd30cbf857bfd7edbd0f9f3eb2b4af86d17c43607a4162b2
|
|
4
|
+
data.tar.gz: 36c38b57abd83bdd5a562581b1ded823e294f3b697341d145f83441a652b0fe9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e3a3976c708fe61aa07898b2aec560032bb3cf1cdb2b65ab3cb492d81fb78418112cfd65b0590255f35f4157cd31e562e3d1f6f86e6a47b96f6c45a4b32d2c09
|
|
7
|
+
data.tar.gz: c9edec290a9e2c3a0b6b3beb6e9404bf1f2a560849ac9786cb2729eaf313028d7530e59e16f7984014f8bd6765160d803ba2fed449010d85e999a26e011c6bd6
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,34 @@
|
|
|
1
1
|
## [Unreleased]
|
|
2
2
|
|
|
3
|
+
## [1.43.0] - 2026-11-09
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
* Add Ruby test automation exclusions ([#669][])
|
|
8
|
+
* Unix domain socket support for Datadog Agent transport ([#678][])
|
|
9
|
+
* knapsack_pro 11 support ([#671][])
|
|
10
|
+
* Cucumber 12 support ([#661][])
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
|
|
14
|
+
* Protect SDK-owned tags and deduplicate CI ENV and git metadata on transport level ([#651][])
|
|
15
|
+
|
|
16
|
+
### Fixed
|
|
17
|
+
|
|
18
|
+
* Fix test retries: retry budgets, suppression reporting, and Cucumber failures ([#673][])
|
|
19
|
+
* Don't fail the test run when a request to the agent fails unexpectedly ([#679][])
|
|
20
|
+
|
|
21
|
+
## [1.42.0] - 2026-10-12
|
|
22
|
+
|
|
23
|
+
### Added
|
|
24
|
+
|
|
25
|
+
* Report Test Impact Analysis skip counts on test suites ([#653][])
|
|
26
|
+
|
|
27
|
+
### Fixed
|
|
28
|
+
|
|
29
|
+
* Fix Minitest tests discovery with Rails 8.1 deferred test loading for Test Parallelization feature ([#655][])
|
|
30
|
+
* Prevent reconfiguration from replacing live CI components ([#644][])
|
|
31
|
+
|
|
3
32
|
## [1.41.0] - 2026-10-05
|
|
4
33
|
|
|
5
34
|
### Fixed
|
|
@@ -737,7 +766,9 @@ Currently test suite level visibility is not used by our instrumentation: it wil
|
|
|
737
766
|
|
|
738
767
|
- Ruby versions < 2.7 no longer supported ([#8][])
|
|
739
768
|
|
|
740
|
-
[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.
|
|
769
|
+
[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.43.0...main
|
|
770
|
+
[1.43.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.42.0...v1.43.0
|
|
771
|
+
[1.42.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.41.0...v1.42.0
|
|
741
772
|
[1.41.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.40.0...v1.41.0
|
|
742
773
|
[1.40.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.39.0...v1.40.0
|
|
743
774
|
[1.39.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.38.1...v1.39.0
|
|
@@ -1042,4 +1073,14 @@ Currently test suite level visibility is not used by our instrumentation: it wil
|
|
|
1042
1073
|
[#633]: https://github.com/DataDog/datadog-ci-rb/issues/633
|
|
1043
1074
|
[#634]: https://github.com/DataDog/datadog-ci-rb/issues/634
|
|
1044
1075
|
[#636]: https://github.com/DataDog/datadog-ci-rb/issues/636
|
|
1045
|
-
[#
|
|
1076
|
+
[#644]: https://github.com/DataDog/datadog-ci-rb/issues/644
|
|
1077
|
+
[#645]: https://github.com/DataDog/datadog-ci-rb/issues/645
|
|
1078
|
+
[#651]: https://github.com/DataDog/datadog-ci-rb/issues/651
|
|
1079
|
+
[#653]: https://github.com/DataDog/datadog-ci-rb/issues/653
|
|
1080
|
+
[#655]: https://github.com/DataDog/datadog-ci-rb/issues/655
|
|
1081
|
+
[#661]: https://github.com/DataDog/datadog-ci-rb/issues/661
|
|
1082
|
+
[#669]: https://github.com/DataDog/datadog-ci-rb/issues/669
|
|
1083
|
+
[#671]: https://github.com/DataDog/datadog-ci-rb/issues/671
|
|
1084
|
+
[#673]: https://github.com/DataDog/datadog-ci-rb/issues/673
|
|
1085
|
+
[#678]: https://github.com/DataDog/datadog-ci-rb/issues/678
|
|
1086
|
+
[#679]: https://github.com/DataDog/datadog-ci-rb/issues/679
|
data/README.md
CHANGED
|
@@ -35,3 +35,48 @@ See [development guide](/docs/DevelopmentGuide.md), [static typing guide](docs/S
|
|
|
35
35
|
## Code of Conduct
|
|
36
36
|
|
|
37
37
|
Everyone interacting in the `Datadog::CI` project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](/CODE_OF_CONDUCT.md).
|
|
38
|
+
|
|
39
|
+
## Test automation exclusions
|
|
40
|
+
|
|
41
|
+
Protect selected tests from Datadog quarantine and disabling with
|
|
42
|
+
`datadog_prevent_failure_suppression`. Exclude selected tests from Datadog Auto
|
|
43
|
+
Test Retries, Early Flake Detection, and Attempt to Fix with
|
|
44
|
+
`datadog_no_auto_retry`. Both declarations may be applied together. They preserve
|
|
45
|
+
explicit skips, Test Impact Analysis selection, and framework-native retries.
|
|
46
|
+
|
|
47
|
+
RSpec uses example metadata:
|
|
48
|
+
|
|
49
|
+
```ruby
|
|
50
|
+
it "checks a critical behavior", datadog_prevent_failure_suppression: true, datadog_no_auto_retry: true do
|
|
51
|
+
expect(critical_behavior).to be_valid
|
|
52
|
+
end
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Minitest exposes per-method declarations after the test method is defined:
|
|
56
|
+
|
|
57
|
+
```ruby
|
|
58
|
+
class CriticalTest < Minitest::Test
|
|
59
|
+
def test_critical_behavior
|
|
60
|
+
assert critical_behavior.valid?
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
datadog_prevent_failure_suppression :test_critical_behavior
|
|
64
|
+
datadog_no_auto_retry :test_critical_behavior
|
|
65
|
+
end
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Undefined method names are ignored with a warning.
|
|
69
|
+
|
|
70
|
+
Cucumber uses scenario tags, including scenario outlines:
|
|
71
|
+
|
|
72
|
+
```gherkin
|
|
73
|
+
@datadog_prevent_failure_suppression @datadog_no_auto_retry
|
|
74
|
+
Scenario: A critical behavior
|
|
75
|
+
Then the critical behavior succeeds
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Load Datadog's instrumentation before defining these tests. Annotated executions
|
|
79
|
+
report `test.test_management.prevent_failure_suppression: true` and/or
|
|
80
|
+
`test.test_management.no_auto_retry: true`. Backend-assigned management tags stay
|
|
81
|
+
visible even when annotations change the effective execution policy. Setting a
|
|
82
|
+
span tag directly does not apply an exclusion.
|
|
@@ -58,10 +58,7 @@ module Datadog
|
|
|
58
58
|
def perform(*events)
|
|
59
59
|
responses = transport.send_events(events)
|
|
60
60
|
|
|
61
|
-
if responses.
|
|
62
|
-
loop_back_off!
|
|
63
|
-
Datadog.logger.warn { "Encountered server error while sending events: #{responses}" }
|
|
64
|
-
end
|
|
61
|
+
loop_back_off! if responses.any?(&:system_error?)
|
|
65
62
|
|
|
66
63
|
nil
|
|
67
64
|
rescue => e
|
|
@@ -39,6 +39,26 @@ module Datadog
|
|
|
39
39
|
synchronize { super }
|
|
40
40
|
end
|
|
41
41
|
|
|
42
|
+
# @internal_api
|
|
43
|
+
def set_internal_tag(key, value)
|
|
44
|
+
synchronize { super }
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# @internal_api
|
|
48
|
+
def set_internal_tags(tags)
|
|
49
|
+
synchronize { super }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# @internal_api
|
|
53
|
+
def set_internal_metric(key, value)
|
|
54
|
+
synchronize { super }
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# @internal_api
|
|
58
|
+
def clear_internal_tag(key)
|
|
59
|
+
synchronize { super }
|
|
60
|
+
end
|
|
61
|
+
|
|
42
62
|
# Finishes the span. This method is thread-safe.
|
|
43
63
|
# @return [void]
|
|
44
64
|
def finish
|
|
@@ -107,7 +107,8 @@ module Datadog
|
|
|
107
107
|
# @type ivar @test_discovery: Datadog::CI::TestDiscovery::Component
|
|
108
108
|
@test_discovery = TestDiscovery::Component.new(
|
|
109
109
|
enabled: settings.ci.test_discovery_enabled,
|
|
110
|
-
output_path: settings.ci.test_discovery_output_path
|
|
110
|
+
output_path: settings.ci.test_discovery_output_path,
|
|
111
|
+
minitest_plugin_autoload_enabled: settings.ci.minitest_no_plugins.nil?
|
|
111
112
|
)
|
|
112
113
|
@test_discovery.disable_features_for_test_discovery!(settings)
|
|
113
114
|
|
|
@@ -5,6 +5,7 @@ require "datadog/core/configuration/components"
|
|
|
5
5
|
|
|
6
6
|
require_relative "settings"
|
|
7
7
|
require_relative "components"
|
|
8
|
+
require_relative "reconfiguration"
|
|
8
9
|
|
|
9
10
|
module Datadog
|
|
10
11
|
module CI
|
|
@@ -14,6 +15,7 @@ module Datadog
|
|
|
14
15
|
def self.activate!
|
|
15
16
|
Core::Configuration::Settings.extend(CI::Configuration::Settings)
|
|
16
17
|
Core::Configuration::Components.prepend(CI::Configuration::Components)
|
|
18
|
+
Datadog.singleton_class.prepend(CI::Configuration::Reconfiguration)
|
|
17
19
|
end
|
|
18
20
|
end
|
|
19
21
|
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Datadog
|
|
4
|
+
module CI
|
|
5
|
+
module Configuration
|
|
6
|
+
# Replacing components during a test session discards its context and remote settings.
|
|
7
|
+
module Reconfiguration
|
|
8
|
+
def configure(&block)
|
|
9
|
+
current = components(allow_initialization: false)
|
|
10
|
+
if current&.test_tracing&.configuration_locked?
|
|
11
|
+
warn_about_ci_reconfiguration
|
|
12
|
+
return configuration
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
super
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
private
|
|
19
|
+
|
|
20
|
+
def warn_about_ci_reconfiguration
|
|
21
|
+
logger.warn(
|
|
22
|
+
"Datadog.configure ignored during an active Test Optimization session to preserve test context and remote settings. " \
|
|
23
|
+
"Move Datadog.configure calls before the test session starts."
|
|
24
|
+
)
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -138,6 +138,7 @@ module Datadog
|
|
|
138
138
|
o.type :int
|
|
139
139
|
o.env CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS
|
|
140
140
|
o.default 5
|
|
141
|
+
o.setter { |value| (value >= 0) ? value : 5 }
|
|
141
142
|
end
|
|
142
143
|
|
|
143
144
|
option :retry_failed_tests_total_limit do |o|
|
|
@@ -190,9 +191,9 @@ module Datadog
|
|
|
190
191
|
end
|
|
191
192
|
|
|
192
193
|
option :test_management_attempt_to_fix_retries_count do |o|
|
|
193
|
-
o.type :int
|
|
194
|
+
o.type :int, nilable: true
|
|
194
195
|
o.env CI::Ext::Settings::ENV_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES
|
|
195
|
-
o.default
|
|
196
|
+
o.default nil
|
|
196
197
|
end
|
|
197
198
|
|
|
198
199
|
option :agentless_logs_submission_enabled do |o|
|
|
@@ -235,6 +236,12 @@ module Datadog
|
|
|
235
236
|
o.env CI::Ext::Settings::ENV_TEST_DISCOVERY_OUTPUT_PATH
|
|
236
237
|
end
|
|
237
238
|
|
|
239
|
+
# Minitest treats any value, including "0", as disabling plugin auto-loading.
|
|
240
|
+
option :minitest_no_plugins do |o|
|
|
241
|
+
o.type :string, nilable: true
|
|
242
|
+
o.env "MT_NO_PLUGINS"
|
|
243
|
+
end
|
|
244
|
+
|
|
238
245
|
option :test_optimization_cache_manifest_file do |o|
|
|
239
246
|
o.type :string, nilable: true
|
|
240
247
|
o.env CI::Ext::TestOptimizationCache::ENV_MANIFEST_FILE
|
|
@@ -22,10 +22,18 @@ module Datadog
|
|
|
22
22
|
@current_test_suite = nil
|
|
23
23
|
|
|
24
24
|
@failed_test_suites_count = 0
|
|
25
|
+
@test_results = {}
|
|
26
|
+
@incomplete_results = false
|
|
25
27
|
|
|
26
28
|
bind_events(config)
|
|
27
29
|
end
|
|
28
30
|
|
|
31
|
+
def failure?
|
|
32
|
+
return nil if @incomplete_results || @test_results.empty?
|
|
33
|
+
|
|
34
|
+
@test_results.value?(CI::Ext::Test::Status::FAIL)
|
|
35
|
+
end
|
|
36
|
+
|
|
29
37
|
def bind_events(config)
|
|
30
38
|
config.on_event :test_run_started, &method(:on_test_run_started)
|
|
31
39
|
config.on_event :test_run_finished, &method(:on_test_run_finished)
|
|
@@ -52,7 +60,8 @@ module Datadog
|
|
|
52
60
|
if event.respond_to?(:success)
|
|
53
61
|
finish_session(event.success)
|
|
54
62
|
else
|
|
55
|
-
|
|
63
|
+
failure = failure?
|
|
64
|
+
finish_session(failure.nil? ? @failed_test_suites_count.zero? : !failure)
|
|
56
65
|
end
|
|
57
66
|
end
|
|
58
67
|
|
|
@@ -87,7 +96,9 @@ module Datadog
|
|
|
87
96
|
event.test_case.name,
|
|
88
97
|
test_suite_name,
|
|
89
98
|
tags: tags,
|
|
90
|
-
service: datadog_configuration[:service_name]
|
|
99
|
+
service: datadog_configuration[:service_name],
|
|
100
|
+
prevent_failure_suppression: event.test_case.match_tags?("@#{CI::Ext::Test::PREVENT_FAILURE_SUPPRESSION_OPTION}"),
|
|
101
|
+
no_auto_retry: event.test_case.match_tags?("@#{CI::Ext::Test::NO_AUTO_RETRY_OPTION}")
|
|
91
102
|
)
|
|
92
103
|
if event.test_case.match_tags?("@#{CI::Ext::Test::ITR_UNSKIPPABLE_OPTION}")
|
|
93
104
|
test_span&.itr_unskippable!
|
|
@@ -96,9 +107,15 @@ module Datadog
|
|
|
96
107
|
|
|
97
108
|
def on_test_case_finished(event)
|
|
98
109
|
test_span = test_tracing_component.active_test
|
|
99
|
-
|
|
110
|
+
if test_span.nil?
|
|
111
|
+
@incomplete_results = true
|
|
112
|
+
return
|
|
113
|
+
end
|
|
100
114
|
|
|
101
115
|
finish_span(test_span, event.result)
|
|
116
|
+
final_status = test_span.get_tag(CI::Ext::Test::TAG_FINAL_STATUS)
|
|
117
|
+
location = event.test_case.location
|
|
118
|
+
@test_results[[location.file, location.line]] = final_status if final_status
|
|
102
119
|
end
|
|
103
120
|
|
|
104
121
|
def on_test_step_started(event)
|
|
@@ -129,7 +146,7 @@ module Datadog
|
|
|
129
146
|
end
|
|
130
147
|
|
|
131
148
|
def finish_span(span, result)
|
|
132
|
-
if !result.passed? && ok?(result
|
|
149
|
+
if !result.passed? && ok?(result)
|
|
133
150
|
span.skipped!(reason: result.message)
|
|
134
151
|
elsif result.passed?
|
|
135
152
|
span.passed!
|
|
@@ -195,13 +212,15 @@ module Datadog
|
|
|
195
212
|
nil
|
|
196
213
|
end
|
|
197
214
|
|
|
198
|
-
def ok?(result
|
|
215
|
+
def ok?(result)
|
|
199
216
|
# in minor update in Cucumber 9.2.0, the arity of the `ok?` method changed
|
|
200
217
|
parameters = result.method(:ok?).parameters
|
|
201
|
-
if parameters
|
|
202
|
-
result.ok?
|
|
218
|
+
if parameters.empty?
|
|
219
|
+
result.ok?
|
|
220
|
+
elsif parameters == [%i[opt be_strict]]
|
|
221
|
+
result.ok?(@config.strict)
|
|
203
222
|
else
|
|
204
|
-
result.ok?(strict: strict)
|
|
223
|
+
result.ok?(strict: @config.strict)
|
|
205
224
|
end
|
|
206
225
|
end
|
|
207
226
|
|
|
@@ -36,6 +36,16 @@ module Datadog
|
|
|
36
36
|
filters_list
|
|
37
37
|
end
|
|
38
38
|
|
|
39
|
+
def failure?
|
|
40
|
+
# Cucumber's summary subtracts a failure for every repeated pass,
|
|
41
|
+
# including EFD and attempt-to-fix passes. Use each test's final
|
|
42
|
+
# result so retries of one scenario cannot erase another failure.
|
|
43
|
+
return super if @configuration.wip? || @configuration.dry_run?
|
|
44
|
+
|
|
45
|
+
failure = datadog_formatter&.failure?
|
|
46
|
+
failure.nil? ? super : failure
|
|
47
|
+
end
|
|
48
|
+
|
|
39
49
|
def begin_scenario(test_case)
|
|
40
50
|
active_suite = Datadog::CI.active_test_suite
|
|
41
51
|
if active_suite&.should_skip?
|
|
@@ -44,9 +54,7 @@ module Datadog
|
|
|
44
54
|
|
|
45
55
|
datadog_test = Datadog::CI.active_test
|
|
46
56
|
|
|
47
|
-
|
|
48
|
-
# attempt_to_fix takes precedence over quarantine: such tests must run (and be retried) even when quarantined.
|
|
49
|
-
if datadog_test&.should_skip? || (datadog_test&.quarantined? && !datadog_test.attempt_to_fix?)
|
|
57
|
+
if datadog_test&.should_skip?
|
|
50
58
|
raise ::Cucumber::Core::Test::Result::Skipped, datadog_test.datadog_skip_reason
|
|
51
59
|
end
|
|
52
60
|
|
|
@@ -40,13 +40,13 @@ module Datadog
|
|
|
40
40
|
set_cookie(CI::Ext::RUM::COOKIE_TEST_EXECUTION_ID, active_test.trace_id.to_s)
|
|
41
41
|
|
|
42
42
|
# set the test type to browser
|
|
43
|
-
active_test.
|
|
43
|
+
active_test.set_internal_tag(CI::Ext::Test::TAG_TYPE, CI::Ext::Test::Type::BROWSER)
|
|
44
44
|
|
|
45
45
|
# set the tags specific to the browser test
|
|
46
|
-
active_test.
|
|
47
|
-
active_test.
|
|
48
|
-
active_test.
|
|
49
|
-
active_test.
|
|
46
|
+
active_test.set_internal_tag(CI::Ext::Test::TAG_BROWSER_DRIVER, "cuprite")
|
|
47
|
+
active_test.set_internal_tag(CI::Ext::Test::TAG_BROWSER_DRIVER_VERSION, datadog_integration.version)
|
|
48
|
+
active_test.set_internal_tag(CI::Ext::Test::TAG_BROWSER_NAME, browser.options.browser_name || "chrome")
|
|
49
|
+
active_test.set_internal_tag(CI::Ext::Test::TAG_BROWSER_VERSION, browser.version.product)
|
|
50
50
|
|
|
51
51
|
result
|
|
52
52
|
end
|
|
@@ -43,7 +43,7 @@ module Datadog
|
|
|
43
43
|
span.failed!
|
|
44
44
|
elsif !test_tracing_component.any_tests_started?
|
|
45
45
|
span.skipped!(reason: "No tests were executed")
|
|
46
|
-
span.
|
|
46
|
+
span.set_internal_tag(CI::Ext::Test::TAG_SESSION_EMPTY_REASON, "zero_tests")
|
|
47
47
|
else
|
|
48
48
|
span.passed!
|
|
49
49
|
end
|
|
@@ -8,6 +8,65 @@ module Datadog
|
|
|
8
8
|
module Minitest
|
|
9
9
|
module Helpers
|
|
10
10
|
module RunnableClassMethods
|
|
11
|
+
# Apply these declarations to named test methods, after defining them.
|
|
12
|
+
def datadog_prevent_failure_suppression(*test_names)
|
|
13
|
+
declarations = (@datadog_prevent_failure_suppression_tests ||= {})
|
|
14
|
+
test_names.each do |test_name|
|
|
15
|
+
declaration = dd_automation_exclusion_declaration(test_name)
|
|
16
|
+
declarations[test_name.to_s] = declaration if declaration
|
|
17
|
+
end
|
|
18
|
+
nil
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def datadog_no_auto_retry(*test_names)
|
|
22
|
+
declarations = (@datadog_no_auto_retry_tests ||= {})
|
|
23
|
+
test_names.each do |test_name|
|
|
24
|
+
declaration = dd_automation_exclusion_declaration(test_name)
|
|
25
|
+
declarations[test_name.to_s] = declaration if declaration
|
|
26
|
+
end
|
|
27
|
+
nil
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def dd_test_prevent_failure_suppression?(test_name, declaration = dd_automation_exclusion_declaration(test_name))
|
|
31
|
+
return false unless declaration
|
|
32
|
+
return true if @datadog_prevent_failure_suppression_tests&.fetch(test_name.to_s, nil) == declaration
|
|
33
|
+
|
|
34
|
+
# Class instance variables are not inherited. Search parents using
|
|
35
|
+
# the child's method identity so overrides cannot inherit exclusions.
|
|
36
|
+
# Steep cannot infer the exclusion API from respond_to?.
|
|
37
|
+
# @type var parent : untyped
|
|
38
|
+
parent = superclass
|
|
39
|
+
parent.respond_to?(:dd_test_prevent_failure_suppression?) && parent.dd_test_prevent_failure_suppression?(test_name, declaration)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def dd_test_no_auto_retry?(test_name, declaration = dd_automation_exclusion_declaration(test_name))
|
|
43
|
+
return false unless declaration
|
|
44
|
+
return true if @datadog_no_auto_retry_tests&.fetch(test_name.to_s, nil) == declaration
|
|
45
|
+
|
|
46
|
+
# Steep cannot infer the exclusion API from respond_to?.
|
|
47
|
+
# @type var parent : untyped
|
|
48
|
+
parent = superclass
|
|
49
|
+
parent.respond_to?(:dd_test_no_auto_retry?) && parent.dd_test_no_auto_retry?(test_name, declaration)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def dd_automation_exclusion_declaration(test_name)
|
|
53
|
+
unless method_defined?(test_name) || private_method_defined?(test_name)
|
|
54
|
+
Datadog.logger.warn("Ignoring Datadog automation exclusion for undefined test method #{self}##{test_name}; define the method before declaring exclusions")
|
|
55
|
+
return nil
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
declaration = instance_method(test_name)
|
|
59
|
+
# On Ruby 2.7, Child.instance_method(:test) != Parent.instance_method(:test)
|
|
60
|
+
# even when Child inherits the same implementation. Resolve through the
|
|
61
|
+
# defining owner so inherited methods match and replacements do not.
|
|
62
|
+
declaration.owner.instance_method(test_name)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def dd_clear_automation_exclusions(test_name)
|
|
66
|
+
@datadog_prevent_failure_suppression_tests&.delete(test_name.to_s)
|
|
67
|
+
@datadog_no_auto_retry_tests&.delete(test_name.to_s)
|
|
68
|
+
end
|
|
69
|
+
|
|
11
70
|
def datadog_itr_unskippable(*args)
|
|
12
71
|
if args.nil? || args.empty?
|
|
13
72
|
@datadog_itr_unskippable_suite = true
|
|
@@ -29,8 +29,8 @@ module Datadog
|
|
|
29
29
|
elsif !test_tracing_component.any_tests_started?
|
|
30
30
|
active_test_module.skipped!(reason: "No tests were executed")
|
|
31
31
|
active_test_session.skipped!(reason: "No tests were executed")
|
|
32
|
-
active_test_module.
|
|
33
|
-
active_test_session.
|
|
32
|
+
active_test_module.set_internal_tag(CI::Ext::Test::TAG_SESSION_EMPTY_REASON, "zero_tests")
|
|
33
|
+
active_test_session.set_internal_tag(CI::Ext::Test::TAG_SESSION_EMPTY_REASON, "zero_tests")
|
|
34
34
|
else
|
|
35
35
|
active_test_module.passed!
|
|
36
36
|
active_test_session.passed!
|
|
@@ -60,8 +60,9 @@ module Datadog
|
|
|
60
60
|
result
|
|
61
61
|
end
|
|
62
62
|
|
|
63
|
-
def run(
|
|
63
|
+
def run(args = [])
|
|
64
64
|
if datadog_configuration[:enabled] && test_discovery_component.enabled?
|
|
65
|
+
prepare_test_discovery(args)
|
|
65
66
|
::Minitest.seed = 1
|
|
66
67
|
discover_tests
|
|
67
68
|
|
|
@@ -72,6 +73,18 @@ module Datadog
|
|
|
72
73
|
|
|
73
74
|
private
|
|
74
75
|
|
|
76
|
+
def prepare_test_discovery(args)
|
|
77
|
+
args = args.dup
|
|
78
|
+
# Minitest 5 auto-loads plugins in run; Minitest 6 uses explicit loading.
|
|
79
|
+
unless ::Minitest.respond_to?(:load) || args.delete("--no-plugins") ||
|
|
80
|
+
!test_discovery_component.minitest_plugin_autoload_enabled?
|
|
81
|
+
::Minitest.load_plugins
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Plugins may load tests while parsing arguments, before execution begins.
|
|
85
|
+
::Minitest.process_args(args)
|
|
86
|
+
end
|
|
87
|
+
|
|
75
88
|
def datadog_integration
|
|
76
89
|
CI::Contrib::Instrumentation.fetch_integration(:minitest)
|
|
77
90
|
end
|
|
@@ -145,7 +145,9 @@ module Datadog
|
|
|
145
145
|
name,
|
|
146
146
|
test_suite_name,
|
|
147
147
|
tags: tags,
|
|
148
|
-
service: datadog_configuration[:service_name]
|
|
148
|
+
service: datadog_configuration[:service_name],
|
|
149
|
+
prevent_failure_suppression: self.class.dd_test_prevent_failure_suppression?(name),
|
|
150
|
+
no_auto_retry: self.class.dd_test_no_auto_retry?(name)
|
|
149
151
|
)
|
|
150
152
|
test_span&.itr_unskippable! if self.class.dd_suite_unskippable? || self.class.dd_test_unskippable?(name)
|
|
151
153
|
|
|
@@ -236,6 +238,7 @@ module Datadog
|
|
|
236
238
|
|
|
237
239
|
module ClassMethods
|
|
238
240
|
def method_added(method_name)
|
|
241
|
+
dd_clear_automation_exclusions(method_name) if respond_to?(:dd_clear_automation_exclusions)
|
|
239
242
|
RunMethodCapture.capture_concrete_pre_datadog_run!(Test, self, InstanceMethods) if method_name == :run
|
|
240
243
|
ensure
|
|
241
244
|
super
|
|
@@ -43,6 +43,8 @@ module Datadog
|
|
|
43
43
|
datadog_test_name,
|
|
44
44
|
datadog_test_suite_name,
|
|
45
45
|
tags: build_test_tags,
|
|
46
|
+
prevent_failure_suppression: datadog_prevent_failure_suppression?,
|
|
47
|
+
no_auto_retry: datadog_no_auto_retry?,
|
|
46
48
|
service: datadog_configuration[:service_name]
|
|
47
49
|
) do |test_span|
|
|
48
50
|
# Set context IDs on the test span for TIA context coverage merging
|
|
@@ -142,6 +144,14 @@ module Datadog
|
|
|
142
144
|
)
|
|
143
145
|
end
|
|
144
146
|
|
|
147
|
+
def datadog_prevent_failure_suppression?
|
|
148
|
+
metadata[CI::Ext::Test::PREVENT_FAILURE_SUPPRESSION_OPTION] == true
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def datadog_no_auto_retry?
|
|
152
|
+
metadata[CI::Ext::Test::NO_AUTO_RETRY_OPTION] == true
|
|
153
|
+
end
|
|
154
|
+
|
|
145
155
|
def datadog_unskippable?
|
|
146
156
|
!!metadata[CI::Ext::Test::ITR_UNSKIPPABLE_OPTION]
|
|
147
157
|
end
|
|
@@ -278,8 +288,8 @@ module Datadog
|
|
|
278
288
|
|
|
279
289
|
update_retry_metadata(test_span) if test_span&.is_retry?
|
|
280
290
|
|
|
281
|
-
metadata[Ext::METADATA_DD_QUARANTINED] =
|
|
282
|
-
metadata[Ext::METADATA_DD_DISABLED] =
|
|
291
|
+
metadata[Ext::METADATA_DD_QUARANTINED] = !!test_span&.should_quarantine?
|
|
292
|
+
metadata[Ext::METADATA_DD_DISABLED] = !!test_span&.should_disable?
|
|
283
293
|
metadata[Ext::METADATA_DD_SKIPPED_BY_ITR] = true if test_span&.skipped_by_test_impact_analysis?
|
|
284
294
|
end
|
|
285
295
|
|
|
@@ -77,14 +77,15 @@ module Datadog
|
|
|
77
77
|
datadog_fqn_test_id = example.datadog_fqn_test_id
|
|
78
78
|
|
|
79
79
|
# Don't skip if the test is marked as attempt_to_fix (it should run and be retried)
|
|
80
|
-
next false if test_management_component&.attempt_to_fix?(datadog_fqn_test_id)
|
|
80
|
+
next false if !example.datadog_no_auto_retry? && test_management_component&.attempt_to_fix?(datadog_fqn_test_id)
|
|
81
81
|
|
|
82
82
|
# Skip by Test Impact Analysis: test is skippable and not marked as unskippable
|
|
83
83
|
tia_skipped = !example.datadog_unskippable? &&
|
|
84
84
|
test_impact_analysis_component&.skippable?(datadog_test_id)
|
|
85
85
|
|
|
86
86
|
# Skip by Test Management: test is disabled
|
|
87
|
-
test_management_disabled =
|
|
87
|
+
test_management_disabled = !example.datadog_prevent_failure_suppression? &&
|
|
88
|
+
test_management_component&.disabled?(datadog_fqn_test_id)
|
|
88
89
|
|
|
89
90
|
tia_skipped || test_management_disabled
|
|
90
91
|
end
|
|
@@ -51,8 +51,8 @@ module Datadog
|
|
|
51
51
|
elsif !test_tracing_component.any_tests_started?
|
|
52
52
|
test_module.skipped!(reason: "No tests were executed")
|
|
53
53
|
test_session.skipped!(reason: "No tests were executed")
|
|
54
|
-
test_module.
|
|
55
|
-
test_session.
|
|
54
|
+
test_module.set_internal_tag(CI::Ext::Test::TAG_SESSION_EMPTY_REASON, "zero_tests")
|
|
55
|
+
test_session.set_internal_tag(CI::Ext::Test::TAG_SESSION_EMPTY_REASON, "zero_tests")
|
|
56
56
|
else
|
|
57
57
|
test_module.passed!
|
|
58
58
|
test_session.passed!
|
|
@@ -37,19 +37,19 @@ module Datadog
|
|
|
37
37
|
@bridge.manage.add_cookie(cookie_hash)
|
|
38
38
|
|
|
39
39
|
# set the test type to browser
|
|
40
|
-
active_test.
|
|
40
|
+
active_test.set_internal_tag(CI::Ext::Test::TAG_TYPE, CI::Ext::Test::Type::BROWSER)
|
|
41
41
|
|
|
42
42
|
# set the tags specific to the browser test
|
|
43
|
-
active_test.
|
|
44
|
-
active_test.
|
|
43
|
+
active_test.set_internal_tag(CI::Ext::Test::TAG_BROWSER_DRIVER, "selenium")
|
|
44
|
+
active_test.set_internal_tag(
|
|
45
45
|
CI::Ext::Test::TAG_BROWSER_DRIVER_VERSION,
|
|
46
46
|
datadog_integration.version
|
|
47
47
|
)
|
|
48
|
-
active_test.
|
|
48
|
+
active_test.set_internal_tag(
|
|
49
49
|
CI::Ext::Test::TAG_BROWSER_NAME,
|
|
50
50
|
@bridge.browser
|
|
51
51
|
)
|
|
52
|
-
active_test.
|
|
52
|
+
active_test.set_internal_tag(
|
|
53
53
|
CI::Ext::Test::TAG_BROWSER_VERSION,
|
|
54
54
|
@bridge.capabilities.browser_version
|
|
55
55
|
)
|