datadog-ci 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +25 -2
- data/README.md +1 -0
- data/ext/datadog_cov/datadog_cov.c +1 -1
- data/lib/datadog/ci/configuration/components.rb +12 -6
- data/lib/datadog/ci/configuration/settings.rb +6 -0
- data/lib/datadog/ci/contrib/cucumber/filter.rb +40 -0
- data/lib/datadog/ci/contrib/cucumber/instrumentation.rb +15 -1
- data/lib/datadog/ci/contrib/cucumber/patcher.rb +0 -2
- data/lib/datadog/ci/contrib/minitest/runner.rb +6 -1
- data/lib/datadog/ci/contrib/minitest/test.rb +4 -0
- data/lib/datadog/ci/contrib/rspec/example.rb +11 -10
- data/lib/datadog/ci/contrib/rspec/runner.rb +2 -1
- data/lib/datadog/ci/ext/settings.rb +1 -0
- data/lib/datadog/ci/ext/telemetry.rb +9 -0
- data/lib/datadog/ci/ext/test.rb +6 -1
- data/lib/datadog/ci/ext/transport.rb +7 -0
- data/lib/datadog/ci/remote/component.rb +13 -2
- data/lib/datadog/ci/remote/library_settings.rb +48 -7
- data/lib/datadog/ci/remote/library_settings_client.rb +2 -1
- data/lib/datadog/ci/remote/slow_test_retries.rb +53 -0
- data/lib/datadog/ci/test.rb +15 -2
- data/lib/datadog/ci/test_optimisation/component.rb +9 -6
- data/lib/datadog/ci/test_optimisation/skippable.rb +1 -1
- data/lib/datadog/ci/test_retries/component.rb +68 -39
- data/lib/datadog/ci/test_retries/driver/base.rb +25 -0
- data/lib/datadog/ci/test_retries/driver/no_retry.rb +16 -0
- data/lib/datadog/ci/test_retries/driver/retry_failed.rb +37 -0
- data/lib/datadog/ci/test_retries/driver/retry_new.rb +50 -0
- data/lib/datadog/ci/test_retries/null_component.rb +7 -6
- data/lib/datadog/ci/test_retries/strategy/base.rb +11 -4
- data/lib/datadog/ci/test_retries/strategy/no_retry.rb +0 -2
- data/lib/datadog/ci/test_retries/strategy/retry_failed.rb +30 -13
- data/lib/datadog/ci/test_retries/strategy/retry_new.rb +132 -0
- data/lib/datadog/ci/test_retries/unique_tests_client.rb +132 -0
- data/lib/datadog/ci/test_session.rb +2 -0
- data/lib/datadog/ci/test_suite.rb +8 -0
- data/lib/datadog/ci/test_visibility/component.rb +23 -13
- data/lib/datadog/ci/test_visibility/null_component.rb +1 -1
- data/lib/datadog/ci/test_visibility/telemetry.rb +9 -0
- data/lib/datadog/ci/utils/test_run.rb +1 -1
- data/lib/datadog/ci/version.rb +1 -1
- data/lib/datadog/ci.rb +6 -3
- metadata +11 -5
- data/lib/datadog/ci/contrib/cucumber/configuration_override.rb +0 -37
- data/lib/datadog/ci/utils/identity.rb +0 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 876a0e7ada3aeaf87b19dade6ee73fec0fe8d37387cde5fc31b43aaad76e99c5
|
4
|
+
data.tar.gz: 8c46ad7c35a621ae03b7b42faa5b3c6934066e422ff4d3a8ac2249cd6466ecd3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8f312e6920cad6072665c00dc5d253e66a7c3272abda20e79f86cf11beb38fffe0be789f644affb6f8920e458f00f221c7969fb84c2ee6f2198af6a140eafc64
|
7
|
+
data.tar.gz: f9b7c2b184e7a140269eaa2069cbf5062270d6bb07410fd9b46cab4c521a8d0bd694a676086164c58300c264357e53e0118611053f6bdf0c2dee757b487b4769
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,21 @@
|
|
1
1
|
## [Unreleased]
|
2
2
|
|
3
|
+
## [1.5.0] - 2024-09-18
|
4
|
+
|
5
|
+
### Added
|
6
|
+
* Retry new tests - parse remote configuration and fetch unique known tests ([#227][])
|
7
|
+
* early flake detection support for rspec and minitest ([#229][])
|
8
|
+
* Early flake detection support for Cucumber ([#231][])
|
9
|
+
|
10
|
+
### Fixed
|
11
|
+
* Minor telemetry fixes ([#226][])
|
12
|
+
|
13
|
+
## [1.4.1] - 2024-08-28
|
14
|
+
|
15
|
+
### Fixed
|
16
|
+
|
17
|
+
* fix datadog_cov crash when doing allocation profiling ([#224][])
|
18
|
+
|
3
19
|
## [1.4.0] - 2024-08-26
|
4
20
|
|
5
21
|
### Added
|
@@ -309,7 +325,9 @@ Currently test suite level visibility is not used by our instrumentation: it wil
|
|
309
325
|
|
310
326
|
- Ruby versions < 2.7 no longer supported ([#8][])
|
311
327
|
|
312
|
-
[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.
|
328
|
+
[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.5.0...main
|
329
|
+
[1.5.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.4.1...v1.5.0
|
330
|
+
[1.4.1]: https://github.com/DataDog/datadog-ci-rb/compare/v1.4.0...v1.4.1
|
313
331
|
[1.4.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.3.0...v1.4.0
|
314
332
|
[1.3.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.2.0...v1.3.0
|
315
333
|
[1.2.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.1.0...v1.2.0
|
@@ -444,4 +462,9 @@ Currently test suite level visibility is not used by our instrumentation: it wil
|
|
444
462
|
[#218]: https://github.com/DataDog/datadog-ci-rb/issues/218
|
445
463
|
[#219]: https://github.com/DataDog/datadog-ci-rb/issues/219
|
446
464
|
[#220]: https://github.com/DataDog/datadog-ci-rb/issues/220
|
447
|
-
[#221]: https://github.com/DataDog/datadog-ci-rb/issues/221
|
465
|
+
[#221]: https://github.com/DataDog/datadog-ci-rb/issues/221
|
466
|
+
[#224]: https://github.com/DataDog/datadog-ci-rb/issues/224
|
467
|
+
[#226]: https://github.com/DataDog/datadog-ci-rb/issues/226
|
468
|
+
[#227]: https://github.com/DataDog/datadog-ci-rb/issues/227
|
469
|
+
[#229]: https://github.com/DataDog/datadog-ci-rb/issues/229
|
470
|
+
[#231]: https://github.com/DataDog/datadog-ci-rb/issues/231
|
data/README.md
CHANGED
@@ -12,6 +12,7 @@ Learn more on our [official website](https://docs.datadoghq.com/tests/) and chec
|
|
12
12
|
|
13
13
|
- [Test Visibility](https://docs.datadoghq.com/tests/) - collect metrics and results for your tests
|
14
14
|
- [Intelligent test runner](https://docs.datadoghq.com/intelligent_test_runner/) - save time by selectively running only tests affected by code changes
|
15
|
+
- [Auto test retries](https://docs.datadoghq.com/tests/auto_test_retries/?tab=ruby) - retrying failing tests up to N times to avoid failing your build due to flaky tests
|
15
16
|
- [Search and manage CI tests](https://docs.datadoghq.com/tests/search/)
|
16
17
|
- [Enhance developer workflows](https://docs.datadoghq.com/tests/developer_workflows)
|
17
18
|
- [Flaky test management](https://docs.datadoghq.com/tests/guides/flaky_test_management/)
|
@@ -249,7 +249,7 @@ static int process_instantiated_klass(st_data_t key, st_data_t _value, st_data_t
|
|
249
249
|
}
|
250
250
|
|
251
251
|
VALUE filename = RARRAY_AREF(source_location, 0);
|
252
|
-
if (filename == Qnil)
|
252
|
+
if (filename == Qnil || !RB_TYPE_P(filename, T_STRING))
|
253
253
|
{
|
254
254
|
return ST_CONTINUE;
|
255
255
|
}
|
@@ -11,6 +11,7 @@ require_relative "../test_optimisation/coverage/transport"
|
|
11
11
|
require_relative "../test_optimisation/coverage/writer"
|
12
12
|
require_relative "../test_retries/component"
|
13
13
|
require_relative "../test_retries/null_component"
|
14
|
+
require_relative "../test_retries/unique_tests_client"
|
14
15
|
require_relative "../test_visibility/component"
|
15
16
|
require_relative "../test_visibility/flush"
|
16
17
|
require_relative "../test_visibility/null_component"
|
@@ -19,7 +20,6 @@ require_relative "../test_visibility/serializers/factories/test_suite_level"
|
|
19
20
|
require_relative "../test_visibility/transport"
|
20
21
|
require_relative "../transport/adapters/telemetry_webmock_safe_adapter"
|
21
22
|
require_relative "../transport/api/builder"
|
22
|
-
require_relative "../utils/identity"
|
23
23
|
require_relative "../utils/parsing"
|
24
24
|
require_relative "../utils/test_run"
|
25
25
|
require_relative "../worker"
|
@@ -116,7 +116,9 @@ module Datadog
|
|
116
116
|
@test_retries = TestRetries::Component.new(
|
117
117
|
retry_failed_tests_enabled: settings.ci.retry_failed_tests_enabled,
|
118
118
|
retry_failed_tests_max_attempts: settings.ci.retry_failed_tests_max_attempts,
|
119
|
-
retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit
|
119
|
+
retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit,
|
120
|
+
retry_new_tests_enabled: settings.ci.retry_new_tests_enabled,
|
121
|
+
unique_tests_client: build_unique_tests_client(settings, test_visibility_api)
|
120
122
|
)
|
121
123
|
# @type ivar @test_optimisation: Datadog::CI::TestOptimisation::Component
|
122
124
|
@test_optimisation = build_test_optimisation(settings, test_visibility_api)
|
@@ -236,6 +238,14 @@ module Datadog
|
|
236
238
|
)
|
237
239
|
end
|
238
240
|
|
241
|
+
def build_unique_tests_client(settings, api)
|
242
|
+
TestRetries::UniqueTestsClient.new(
|
243
|
+
api: api,
|
244
|
+
dd_env: settings.env,
|
245
|
+
config_tags: custom_configuration(settings)
|
246
|
+
)
|
247
|
+
end
|
248
|
+
|
239
249
|
# fetch custom tags provided by the user in DD_TAGS env var
|
240
250
|
# with prefix test.configuration.
|
241
251
|
def custom_configuration(settings)
|
@@ -274,12 +284,8 @@ module Datadog
|
|
274
284
|
settings.telemetry.shutdown_timeout_seconds = 60.0
|
275
285
|
|
276
286
|
begin
|
277
|
-
require "datadog/core/environment/identity"
|
278
287
|
require "datadog/core/telemetry/http/adapters/net"
|
279
288
|
|
280
|
-
# patch gem's identity to report datadog-ci library version instead of datadog gem version
|
281
|
-
Core::Environment::Identity.include(CI::Utils::Identity)
|
282
|
-
|
283
289
|
# patch gem's telemetry transport layer to use Net::HTTP instead of WebMock's Net::HTTP
|
284
290
|
Core::Telemetry::Http::Adapters::Net.include(CI::Transport::Adapters::TelemetryWebmockSafeAdapter)
|
285
291
|
rescue => e
|
@@ -106,6 +106,12 @@ module Datadog
|
|
106
106
|
o.default 1000
|
107
107
|
end
|
108
108
|
|
109
|
+
option :retry_new_tests_enabled do |o|
|
110
|
+
o.type :bool
|
111
|
+
o.env CI::Ext::Settings::ENV_RETRY_NEW_TESTS_ENABLED
|
112
|
+
o.default true
|
113
|
+
end
|
114
|
+
|
109
115
|
define_method(:instrument) do |integration_name, options = {}, &block|
|
110
116
|
return unless enabled
|
111
117
|
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Datadog
|
4
|
+
module CI
|
5
|
+
module Contrib
|
6
|
+
module Cucumber
|
7
|
+
class Filter < ::Cucumber::Core::Filter.new(:configuration)
|
8
|
+
def test_case(test_case)
|
9
|
+
test_retries_component.reset_retries! unless test_case_seen[test_case]
|
10
|
+
test_case_seen[test_case] = true
|
11
|
+
|
12
|
+
configuration.on_event(:test_case_finished) do |event|
|
13
|
+
next unless retry_required?(test_case, event)
|
14
|
+
|
15
|
+
test_case.describe_to(receiver)
|
16
|
+
end
|
17
|
+
|
18
|
+
super
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def retry_required?(test_case, event)
|
24
|
+
return false unless event.test_case == test_case
|
25
|
+
|
26
|
+
test_retries_component.should_retry?
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_case_seen
|
30
|
+
@test_case_seen ||= Hash.new { |h, k| h[k] = false }
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_retries_component
|
34
|
+
@test_retries_component ||= Datadog.send(:components).test_retries
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -18,10 +18,24 @@ module Datadog
|
|
18
18
|
|
19
19
|
def formatters
|
20
20
|
existing_formatters = super
|
21
|
-
@datadog_formatter ||=
|
21
|
+
@datadog_formatter ||= Formatter.new(@configuration)
|
22
22
|
[@datadog_formatter] + existing_formatters
|
23
23
|
end
|
24
24
|
|
25
|
+
def filters
|
26
|
+
require_relative "filter"
|
27
|
+
|
28
|
+
filters_list = super
|
29
|
+
datadog_filter = Filter.new(@configuration)
|
30
|
+
unless @configuration.dry_run?
|
31
|
+
# insert our filter the pre-last position because Cucumber::Filters::PrepareWorld must be the last one
|
32
|
+
# see:
|
33
|
+
# https://github.com/cucumber/cucumber-ruby/blob/58dd8f12c0ac5f4e607335ff2e7d385c1ed25899/lib/cucumber/runtime.rb#L266
|
34
|
+
filters_list.insert(-2, datadog_filter)
|
35
|
+
end
|
36
|
+
filters_list
|
37
|
+
end
|
38
|
+
|
25
39
|
def begin_scenario(test_case)
|
26
40
|
if Datadog::CI.active_test&.skipped_by_itr?
|
27
41
|
raise ::Cucumber::Core::Test::Result::Skipped, CI::Ext::Test::ITR_TEST_SKIP_REASON
|
@@ -3,7 +3,6 @@
|
|
3
3
|
require "datadog/tracing/contrib/patcher"
|
4
4
|
|
5
5
|
require_relative "instrumentation"
|
6
|
-
require_relative "configuration_override"
|
7
6
|
|
8
7
|
module Datadog
|
9
8
|
module CI
|
@@ -21,7 +20,6 @@ module Datadog
|
|
21
20
|
|
22
21
|
def patch
|
23
22
|
::Cucumber::Runtime.include(Instrumentation)
|
24
|
-
::Cucumber::Configuration.include(ConfigurationOverride)
|
25
23
|
end
|
26
24
|
end
|
27
25
|
end
|
@@ -8,6 +8,8 @@ module Datadog
|
|
8
8
|
module Contrib
|
9
9
|
module Minitest
|
10
10
|
module Runner
|
11
|
+
DD_ESTIMATED_TESTS_PER_SUITE = 5
|
12
|
+
|
11
13
|
def self.included(base)
|
12
14
|
base.singleton_class.prepend(ClassMethods)
|
13
15
|
end
|
@@ -18,12 +20,15 @@ module Datadog
|
|
18
20
|
|
19
21
|
return unless datadog_configuration[:enabled]
|
20
22
|
|
23
|
+
# minitest does not store the total number of tests, so we can't pass it to the test session
|
24
|
+
# instead, we use the number of test suites * DD_ESTIMATED_TESTS_PER_SUITE as a rough estimate
|
21
25
|
test_visibility_component.start_test_session(
|
22
26
|
tags: {
|
23
27
|
CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
|
24
28
|
CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s
|
25
29
|
},
|
26
|
-
service: datadog_configuration[:service_name]
|
30
|
+
service: datadog_configuration[:service_name],
|
31
|
+
total_tests_count: (DD_ESTIMATED_TESTS_PER_SUITE * ::Minitest::Runnable.runnables.size).to_i
|
27
32
|
)
|
28
33
|
test_visibility_component.start_test_module(Ext::FRAMEWORK)
|
29
34
|
end
|
@@ -51,6 +51,10 @@ module Datadog
|
|
51
51
|
return super unless test_span
|
52
52
|
|
53
53
|
finish_with_result(test_span, result_code)
|
54
|
+
|
55
|
+
# remove failures if test passed at least once on retries
|
56
|
+
self.failures = [] if test_span.any_retry_passed?
|
57
|
+
|
54
58
|
if Helpers.parallel?(self.class)
|
55
59
|
finish_with_result(test_span.test_suite, result_code)
|
56
60
|
end
|
@@ -64,9 +64,9 @@ module Datadog
|
|
64
64
|
|
65
65
|
result = super
|
66
66
|
|
67
|
-
#
|
67
|
+
# When test job is canceled and RSpec is quitting we don't want to report the last test
|
68
68
|
# before RSpec context unwinds. This test might have some unrelated errors that we don't want to
|
69
|
-
#
|
69
|
+
# see in Datadog.
|
70
70
|
return result if ::RSpec.world.wants_to_quit
|
71
71
|
|
72
72
|
case execution_result.status
|
@@ -74,6 +74,8 @@ module Datadog
|
|
74
74
|
test_span&.passed!
|
75
75
|
when :failed
|
76
76
|
test_span&.failed!(exception: execution_result.exception)
|
77
|
+
# if any of the retries passed, we don't fail the test run
|
78
|
+
@exception = nil if test_span&.any_retry_passed?
|
77
79
|
else
|
78
80
|
# :pending or nil
|
79
81
|
test_span&.skipped!(
|
@@ -84,21 +86,20 @@ module Datadog
|
|
84
86
|
end
|
85
87
|
end
|
86
88
|
|
87
|
-
# after retries are done, we can report the test to RSpec
|
88
|
-
@skip_reporting = false
|
89
|
-
|
90
89
|
# this is a special case for ci-queue, we need to finish the test suite span
|
91
90
|
ci_queue_test_span&.finish
|
92
91
|
|
93
|
-
#
|
94
|
-
|
95
|
-
# if test passed at least once
|
92
|
+
# after retries are done, we can finally report the test to RSpec
|
93
|
+
@skip_reporting = false
|
96
94
|
finish(reporter)
|
97
95
|
end
|
98
96
|
|
99
97
|
def finish(reporter)
|
100
|
-
#
|
101
|
-
# it is going to be reported once after retries are done
|
98
|
+
# By default finish test but do not report it to RSpec::Core::Reporter
|
99
|
+
# it is going to be reported once after retries are done.
|
100
|
+
#
|
101
|
+
# We need to do this because RSpec breaks when we try to report the same example multiple times with different
|
102
|
+
# results.
|
102
103
|
return super unless @skip_reporting
|
103
104
|
|
104
105
|
super(::RSpec::Core::NullReporter)
|
@@ -23,7 +23,8 @@ module Datadog
|
|
23
23
|
CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
|
24
24
|
CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s
|
25
25
|
},
|
26
|
-
service: datadog_configuration[:service_name]
|
26
|
+
service: datadog_configuration[:service_name],
|
27
|
+
total_tests_count: ::RSpec.world.example_count
|
27
28
|
)
|
28
29
|
|
29
30
|
test_module = test_visibility_component.start_test_module(Ext::FRAMEWORK)
|
@@ -18,6 +18,7 @@ module Datadog
|
|
18
18
|
ENV_RETRY_FAILED_TESTS_ENABLED = "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED"
|
19
19
|
ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS = "DD_CIVISIBILITY_FLAKY_RETRY_COUNT"
|
20
20
|
ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT = "DD_CIVISIBILITY_TOTAL_FLAKY_RETRY_COUNT"
|
21
|
+
ENV_RETRY_NEW_TESTS_ENABLED = "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED"
|
21
22
|
|
22
23
|
# Source: https://docs.datadoghq.com/getting_started/site/
|
23
24
|
DD_SITE_ALLOWLIST = %w[
|
@@ -54,6 +54,12 @@ module Datadog
|
|
54
54
|
METRIC_CODE_COVERAGE_IS_EMPTY = "code_coverage.is_empty"
|
55
55
|
METRIC_CODE_COVERAGE_FILES = "code_coverage.files"
|
56
56
|
|
57
|
+
METRIC_EFD_UNIQUE_TESTS_REQUEST = "early_flake_detection.request"
|
58
|
+
METRIC_EFD_UNIQUE_TESTS_REQUEST_MS = "early_flake_detection.request_ms"
|
59
|
+
METRIC_EFD_UNIQUE_TESTS_REQUEST_ERRORS = "early_flake_detection.request_errors"
|
60
|
+
METRIC_EFD_UNIQUE_TESTS_RESPONSE_BYTES = "early_flake_detection.response_bytes"
|
61
|
+
METRIC_EFD_UNIQUE_TESTS_RESPONSE_TESTS = "early_flake_detection.response_tests"
|
62
|
+
|
57
63
|
METRIC_TEST_SESSION = "test_session"
|
58
64
|
|
59
65
|
TAG_TEST_FRAMEWORK = "test_framework"
|
@@ -63,6 +69,7 @@ module Datadog
|
|
63
69
|
TAG_BROWSER_DRIVER = "browser_driver"
|
64
70
|
TAG_IS_RUM = "is_rum"
|
65
71
|
TAG_IS_RETRY = "is_retry"
|
72
|
+
TAG_IS_NEW = "is_new"
|
66
73
|
TAG_LIBRARY = "library"
|
67
74
|
TAG_ENDPOINT = "endpoint"
|
68
75
|
TAG_ERROR_TYPE = "error_type"
|
@@ -73,6 +80,8 @@ module Datadog
|
|
73
80
|
TAG_COMMAND = "command"
|
74
81
|
TAG_COVERAGE_ENABLED = "coverage_enabled"
|
75
82
|
TAG_ITR_SKIP_ENABLED = "itrskip_enabled"
|
83
|
+
TAG_EARLY_FLAKE_DETECTION_ENABLED = "early_flake_detection_enabled"
|
84
|
+
TAG_EARLY_FLAKE_DETECTION_ABORT_REASON = "early_flake_detection_abort_reason"
|
76
85
|
TAG_PROVIDER = "provider"
|
77
86
|
TAG_AUTO_INJECTED = "auto_injected"
|
78
87
|
|
data/lib/datadog/ci/ext/test.rb
CHANGED
@@ -58,8 +58,11 @@ module Datadog
|
|
58
58
|
# version of the browser, if multiple browsers or multiple versions then this tag is empty
|
59
59
|
TAG_BROWSER_VERSION = "test.browser.version"
|
60
60
|
|
61
|
-
# Tags for
|
61
|
+
# Tags for retries
|
62
62
|
TAG_IS_RETRY = "test.is_retry" # true if test was retried by datadog-ci library
|
63
|
+
TAG_IS_NEW = "test.is_new" # true if test was marked as new by new test retries (early flake detection)
|
64
|
+
TAG_EARLY_FLAKE_ENABLED = "test.early_flake.enabled" # true if early flake detection is enabled
|
65
|
+
TAG_EARLY_FLAKE_ABORT_REASON = "test.early_flake.abort_reason" # reason why early flake detection was aborted
|
63
66
|
|
64
67
|
# internal APM tag to mark a span as a test span
|
65
68
|
TAG_SPAN_KIND = "span.kind"
|
@@ -73,6 +76,8 @@ module Datadog
|
|
73
76
|
ITR_TEST_SKIP_REASON = "Skipped by Datadog's intelligent test runner"
|
74
77
|
ITR_UNSKIPPABLE_OPTION = :datadog_itr_unskippable
|
75
78
|
|
79
|
+
EARLY_FLAKE_FAULTY = "faulty"
|
80
|
+
|
76
81
|
# test status as recognized by Datadog
|
77
82
|
module Status
|
78
83
|
PASS = "pass"
|
@@ -37,6 +37,10 @@ module Datadog
|
|
37
37
|
DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY = "tests_skipping"
|
38
38
|
DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY = "require_git"
|
39
39
|
DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY = "flaky_test_retries_enabled"
|
40
|
+
DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY = "early_flake_detection"
|
41
|
+
DD_API_SETTINGS_RESPONSE_ENABLED_KEY = "enabled"
|
42
|
+
DD_API_SETTINGS_RESPONSE_SLOW_TEST_RETRIES_KEY = "slow_test_retries"
|
43
|
+
DD_API_SETTINGS_RESPONSE_FAULTY_SESSION_THRESHOLD_KEY = "faulty_session_threshold"
|
40
44
|
DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze
|
41
45
|
|
42
46
|
DD_API_GIT_SEARCH_COMMITS_PATH = "/api/v2/git/repository/search_commits"
|
@@ -46,6 +50,9 @@ module Datadog
|
|
46
50
|
DD_API_SKIPPABLE_TESTS_PATH = "/api/v2/ci/tests/skippable"
|
47
51
|
DD_API_SKIPPABLE_TESTS_TYPE = "test_params"
|
48
52
|
|
53
|
+
DD_API_UNIQUE_TESTS_PATH = "/api/v2/ci/libraries/tests"
|
54
|
+
DD_API_UNIQUE_TESTS_TYPE = "ci_app_libraries_tests_request"
|
55
|
+
|
49
56
|
CONTENT_TYPE_MESSAGEPACK = "application/msgpack"
|
50
57
|
CONTENT_TYPE_JSON = "application/json"
|
51
58
|
CONTENT_TYPE_MULTIPART_FORM_DATA = "multipart/form-data"
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative "../worker"
|
4
|
+
|
3
5
|
module Datadog
|
4
6
|
module CI
|
5
7
|
module Remote
|
@@ -27,8 +29,17 @@ module Datadog
|
|
27
29
|
end
|
28
30
|
end
|
29
31
|
|
30
|
-
|
31
|
-
|
32
|
+
# configure different components in parallel because they might block on HTTP requests
|
33
|
+
configuration_workers = [
|
34
|
+
Worker.new { test_optimisation.configure(library_configuration, test_session) },
|
35
|
+
Worker.new { test_retries.configure(library_configuration, test_session) }
|
36
|
+
]
|
37
|
+
|
38
|
+
# launch configuration workers
|
39
|
+
configuration_workers.each(&:perform)
|
40
|
+
|
41
|
+
# block until all workers are done (or 60 seconds has passed)
|
42
|
+
configuration_workers.each(&:wait_until_done)
|
32
43
|
end
|
33
44
|
|
34
45
|
private
|
@@ -7,6 +7,8 @@ require_relative "../ext/transport"
|
|
7
7
|
require_relative "../transport/telemetry"
|
8
8
|
require_relative "../utils/parsing"
|
9
9
|
|
10
|
+
require_relative "slow_test_retries"
|
11
|
+
|
10
12
|
module Datadog
|
11
13
|
module CI
|
12
14
|
module Remote
|
@@ -49,37 +51,76 @@ module Datadog
|
|
49
51
|
def require_git?
|
50
52
|
return @require_git if defined?(@require_git)
|
51
53
|
|
52
|
-
@require_git =
|
54
|
+
@require_git = Utils::Parsing.convert_to_bool(
|
55
|
+
payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY, false)
|
56
|
+
)
|
53
57
|
end
|
54
58
|
|
55
59
|
def itr_enabled?
|
56
60
|
return @itr_enabled if defined?(@itr_enabled)
|
57
61
|
|
58
|
-
@itr_enabled =
|
62
|
+
@itr_enabled = Utils::Parsing.convert_to_bool(
|
63
|
+
payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY, false)
|
64
|
+
)
|
59
65
|
end
|
60
66
|
|
61
67
|
def code_coverage_enabled?
|
62
68
|
return @code_coverage_enabled if defined?(@code_coverage_enabled)
|
63
69
|
|
64
|
-
@code_coverage_enabled =
|
70
|
+
@code_coverage_enabled = Utils::Parsing.convert_to_bool(
|
71
|
+
payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY, false)
|
72
|
+
)
|
65
73
|
end
|
66
74
|
|
67
75
|
def tests_skipping_enabled?
|
68
76
|
return @tests_skipping_enabled if defined?(@tests_skipping_enabled)
|
69
77
|
|
70
|
-
@tests_skipping_enabled =
|
78
|
+
@tests_skipping_enabled = Utils::Parsing.convert_to_bool(
|
79
|
+
payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY, false)
|
80
|
+
)
|
71
81
|
end
|
72
82
|
|
73
83
|
def flaky_test_retries_enabled?
|
74
84
|
return @flaky_test_retries_enabled if defined?(@flaky_test_retries_enabled)
|
75
85
|
|
76
|
-
@flaky_test_retries_enabled =
|
86
|
+
@flaky_test_retries_enabled = Utils::Parsing.convert_to_bool(
|
87
|
+
payload.fetch(
|
88
|
+
Ext::Transport::DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY, false
|
89
|
+
)
|
90
|
+
)
|
91
|
+
end
|
92
|
+
|
93
|
+
def early_flake_detection_enabled?
|
94
|
+
return @early_flake_detection_enabled if defined?(@early_flake_detection_enabled)
|
95
|
+
|
96
|
+
@early_flake_detection_enabled = Utils::Parsing.convert_to_bool(
|
97
|
+
early_flake_detection_payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_ENABLED_KEY, false)
|
98
|
+
)
|
99
|
+
end
|
100
|
+
|
101
|
+
def slow_test_retries
|
102
|
+
return @slow_test_retries if defined?(@slow_test_retries)
|
103
|
+
|
104
|
+
@slow_test_retries = SlowTestRetries.new(
|
105
|
+
early_flake_detection_payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_SLOW_TEST_RETRIES_KEY, {})
|
106
|
+
)
|
107
|
+
end
|
108
|
+
|
109
|
+
def faulty_session_threshold
|
110
|
+
return @faulty_session_threshold if defined?(@faulty_session_threshold)
|
111
|
+
|
112
|
+
@faulty_session_threshold = early_flake_detection_payload.fetch(
|
113
|
+
Ext::Transport::DD_API_SETTINGS_RESPONSE_FAULTY_SESSION_THRESHOLD_KEY, 0
|
114
|
+
)
|
77
115
|
end
|
78
116
|
|
79
117
|
private
|
80
118
|
|
81
|
-
def
|
82
|
-
|
119
|
+
def early_flake_detection_payload
|
120
|
+
payload.fetch(
|
121
|
+
Ext::Transport::DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY,
|
122
|
+
{}
|
123
|
+
)
|
83
124
|
end
|
84
125
|
|
85
126
|
def default_payload
|
@@ -58,7 +58,8 @@ module Datadog
|
|
58
58
|
1,
|
59
59
|
{
|
60
60
|
Ext::Telemetry::TAG_COVERAGE_ENABLED => library_settings.code_coverage_enabled?.to_s,
|
61
|
-
Ext::Telemetry::TAG_ITR_SKIP_ENABLED => library_settings.tests_skipping_enabled?.to_s
|
61
|
+
Ext::Telemetry::TAG_ITR_SKIP_ENABLED => library_settings.tests_skipping_enabled?.to_s,
|
62
|
+
Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ENABLED => library_settings.early_flake_detection_enabled?.to_s
|
62
63
|
}
|
63
64
|
)
|
64
65
|
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Datadog
|
4
|
+
module CI
|
5
|
+
module Remote
|
6
|
+
# Parses "slow_test_retries" payload for early flake detection settings
|
7
|
+
#
|
8
|
+
# Example payload:
|
9
|
+
# {
|
10
|
+
# "5s" => 10,
|
11
|
+
# "10s" => 5,
|
12
|
+
# "30s" => 3,
|
13
|
+
# "5m" => 2
|
14
|
+
# }
|
15
|
+
#
|
16
|
+
# The payload above means that for tests that run less than 5 seconds, we should retry them 10 times,
|
17
|
+
# for tests that run less than 10 seconds, we should retry them 5 times, and so on.
|
18
|
+
class SlowTestRetries
|
19
|
+
attr_reader :entries
|
20
|
+
|
21
|
+
Entry = Struct.new(:duration, :max_attempts)
|
22
|
+
|
23
|
+
DURATION_MEASURES = {
|
24
|
+
"s" => 1,
|
25
|
+
"m" => 60
|
26
|
+
}.freeze
|
27
|
+
|
28
|
+
def initialize(payload)
|
29
|
+
@entries = parse(payload)
|
30
|
+
end
|
31
|
+
|
32
|
+
def max_attempts_for_duration(duration)
|
33
|
+
@entries.each do |entry|
|
34
|
+
return entry.max_attempts if duration < entry.duration
|
35
|
+
end
|
36
|
+
|
37
|
+
0
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def parse(payload)
|
43
|
+
(payload || {}).keys.filter_map do |key|
|
44
|
+
duration, measure = key.match(/(\d+)(\w+)/)&.captures
|
45
|
+
next if duration.nil? || measure.nil? || !DURATION_MEASURES.key?(measure)
|
46
|
+
|
47
|
+
Entry.new(duration.to_f * DURATION_MEASURES.fetch(measure, 1), payload[key].to_i)
|
48
|
+
end.sort_by(&:duration)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
data/lib/datadog/ci/test.rb
CHANGED
@@ -70,6 +70,12 @@ module Datadog
|
|
70
70
|
get_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR) == "true"
|
71
71
|
end
|
72
72
|
|
73
|
+
# Returns "true" if test span represents a retry.
|
74
|
+
# @return [Boolean] true if this test is a retry, false otherwise.
|
75
|
+
def is_retry?
|
76
|
+
get_tag(Ext::Test::TAG_IS_RETRY) == "true"
|
77
|
+
end
|
78
|
+
|
73
79
|
# Marks this test as unskippable by the intelligent test runner.
|
74
80
|
# This must be done before the test execution starts.
|
75
81
|
#
|
@@ -139,11 +145,18 @@ module Datadog
|
|
139
145
|
get_tag(Ext::Test::TAG_PARAMETERS)
|
140
146
|
end
|
141
147
|
|
148
|
+
# @internal
|
149
|
+
def any_retry_passed?
|
150
|
+
!!test_suite&.any_test_retry_passed?(test_id)
|
151
|
+
end
|
152
|
+
|
142
153
|
private
|
143
154
|
|
144
|
-
def
|
145
|
-
test_id
|
155
|
+
def test_id
|
156
|
+
@test_id ||= Utils::TestRun.datadog_test_id(name, test_suite_name, parameters)
|
157
|
+
end
|
146
158
|
|
159
|
+
def record_test_result(datadog_status)
|
147
160
|
# if this test was already executed in this test suite, mark it as retried
|
148
161
|
if test_suite&.test_executed?(test_id)
|
149
162
|
set_tag(Ext::Test::TAG_IS_RETRY, "true")
|