datadog-ci 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +25 -2
  3. data/README.md +1 -0
  4. data/ext/datadog_cov/datadog_cov.c +1 -1
  5. data/lib/datadog/ci/configuration/components.rb +12 -6
  6. data/lib/datadog/ci/configuration/settings.rb +6 -0
  7. data/lib/datadog/ci/contrib/cucumber/filter.rb +40 -0
  8. data/lib/datadog/ci/contrib/cucumber/instrumentation.rb +15 -1
  9. data/lib/datadog/ci/contrib/cucumber/patcher.rb +0 -2
  10. data/lib/datadog/ci/contrib/minitest/runner.rb +6 -1
  11. data/lib/datadog/ci/contrib/minitest/test.rb +4 -0
  12. data/lib/datadog/ci/contrib/rspec/example.rb +11 -10
  13. data/lib/datadog/ci/contrib/rspec/runner.rb +2 -1
  14. data/lib/datadog/ci/ext/settings.rb +1 -0
  15. data/lib/datadog/ci/ext/telemetry.rb +9 -0
  16. data/lib/datadog/ci/ext/test.rb +6 -1
  17. data/lib/datadog/ci/ext/transport.rb +7 -0
  18. data/lib/datadog/ci/remote/component.rb +13 -2
  19. data/lib/datadog/ci/remote/library_settings.rb +48 -7
  20. data/lib/datadog/ci/remote/library_settings_client.rb +2 -1
  21. data/lib/datadog/ci/remote/slow_test_retries.rb +53 -0
  22. data/lib/datadog/ci/test.rb +15 -2
  23. data/lib/datadog/ci/test_optimisation/component.rb +9 -6
  24. data/lib/datadog/ci/test_optimisation/skippable.rb +1 -1
  25. data/lib/datadog/ci/test_retries/component.rb +68 -39
  26. data/lib/datadog/ci/test_retries/driver/base.rb +25 -0
  27. data/lib/datadog/ci/test_retries/driver/no_retry.rb +16 -0
  28. data/lib/datadog/ci/test_retries/driver/retry_failed.rb +37 -0
  29. data/lib/datadog/ci/test_retries/driver/retry_new.rb +50 -0
  30. data/lib/datadog/ci/test_retries/null_component.rb +7 -6
  31. data/lib/datadog/ci/test_retries/strategy/base.rb +11 -4
  32. data/lib/datadog/ci/test_retries/strategy/no_retry.rb +0 -2
  33. data/lib/datadog/ci/test_retries/strategy/retry_failed.rb +30 -13
  34. data/lib/datadog/ci/test_retries/strategy/retry_new.rb +132 -0
  35. data/lib/datadog/ci/test_retries/unique_tests_client.rb +132 -0
  36. data/lib/datadog/ci/test_session.rb +2 -0
  37. data/lib/datadog/ci/test_suite.rb +8 -0
  38. data/lib/datadog/ci/test_visibility/component.rb +23 -13
  39. data/lib/datadog/ci/test_visibility/null_component.rb +1 -1
  40. data/lib/datadog/ci/test_visibility/telemetry.rb +9 -0
  41. data/lib/datadog/ci/utils/test_run.rb +1 -1
  42. data/lib/datadog/ci/version.rb +1 -1
  43. data/lib/datadog/ci.rb +6 -3
  44. metadata +11 -5
  45. data/lib/datadog/ci/contrib/cucumber/configuration_override.rb +0 -37
  46. data/lib/datadog/ci/utils/identity.rb +0 -20
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 614d612dad9b7e29104499934b7e80b44a0cd5f60ec5135593a3bccd9df5fc9c
4
- data.tar.gz: bb498cfa7db7871f2ac03aedb2dbbb94fac878ea34ff33d0a205e216fe856735
3
+ metadata.gz: 876a0e7ada3aeaf87b19dade6ee73fec0fe8d37387cde5fc31b43aaad76e99c5
4
+ data.tar.gz: 8c46ad7c35a621ae03b7b42faa5b3c6934066e422ff4d3a8ac2249cd6466ecd3
5
5
  SHA512:
6
- metadata.gz: 7edc6574a2a647e49da0c5ae487f98c89fa097aae064cbfd42b2a08b88ae1611fee948632dca90c6b2bb9166e5f3e02ea3aaffeb0f419f6f84f3905eac384b53
7
- data.tar.gz: 815bdb39b87e95ee1a8fe7ed5c9ae9b545491ff0c59beb117a7ff0202b73aede9c14ee4fc5179195d726ed4f4dd0572de95f3bb795f26124062f332e138e8bbe
6
+ metadata.gz: 8f312e6920cad6072665c00dc5d253e66a7c3272abda20e79f86cf11beb38fffe0be789f644affb6f8920e458f00f221c7969fb84c2ee6f2198af6a140eafc64
7
+ data.tar.gz: f9b7c2b184e7a140269eaa2069cbf5062270d6bb07410fd9b46cab4c521a8d0bd694a676086164c58300c264357e53e0118611053f6bdf0c2dee757b487b4769
data/CHANGELOG.md CHANGED
@@ -1,5 +1,21 @@
1
1
  ## [Unreleased]
2
2
 
3
+ ## [1.5.0] - 2024-09-18
4
+
5
+ ### Added
6
+ * Retry new tests - parse remote configuration and fetch unique known tests ([#227][])
7
+ * early flake detection support for rspec and minitest ([#229][])
8
+ * Early flake detection support for Cucumber ([#231][])
9
+
10
+ ### Fixed
11
+ * Minor telemetry fixes ([#226][])
12
+
13
+ ## [1.4.1] - 2024-08-28
14
+
15
+ ### Fixed
16
+
17
+ * fix datadog_cov crash when doing allocation profiling ([#224][])
18
+
3
19
  ## [1.4.0] - 2024-08-26
4
20
 
5
21
  ### Added
@@ -309,7 +325,9 @@ Currently test suite level visibility is not used by our instrumentation: it wil
309
325
 
310
326
  - Ruby versions < 2.7 no longer supported ([#8][])
311
327
 
312
- [Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.4.0...main
328
+ [Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.5.0...main
329
+ [1.5.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.4.1...v1.5.0
330
+ [1.4.1]: https://github.com/DataDog/datadog-ci-rb/compare/v1.4.0...v1.4.1
313
331
  [1.4.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.3.0...v1.4.0
314
332
  [1.3.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.2.0...v1.3.0
315
333
  [1.2.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.1.0...v1.2.0
@@ -444,4 +462,9 @@ Currently test suite level visibility is not used by our instrumentation: it wil
444
462
  [#218]: https://github.com/DataDog/datadog-ci-rb/issues/218
445
463
  [#219]: https://github.com/DataDog/datadog-ci-rb/issues/219
446
464
  [#220]: https://github.com/DataDog/datadog-ci-rb/issues/220
447
- [#221]: https://github.com/DataDog/datadog-ci-rb/issues/221
465
+ [#221]: https://github.com/DataDog/datadog-ci-rb/issues/221
466
+ [#224]: https://github.com/DataDog/datadog-ci-rb/issues/224
467
+ [#226]: https://github.com/DataDog/datadog-ci-rb/issues/226
468
+ [#227]: https://github.com/DataDog/datadog-ci-rb/issues/227
469
+ [#229]: https://github.com/DataDog/datadog-ci-rb/issues/229
470
+ [#231]: https://github.com/DataDog/datadog-ci-rb/issues/231
data/README.md CHANGED
@@ -12,6 +12,7 @@ Learn more on our [official website](https://docs.datadoghq.com/tests/) and chec
12
12
 
13
13
  - [Test Visibility](https://docs.datadoghq.com/tests/) - collect metrics and results for your tests
14
14
  - [Intelligent test runner](https://docs.datadoghq.com/intelligent_test_runner/) - save time by selectively running only tests affected by code changes
15
+ - [Auto test retries](https://docs.datadoghq.com/tests/auto_test_retries/?tab=ruby) - retrying failing tests up to N times to avoid failing your build due to flaky tests
15
16
  - [Search and manage CI tests](https://docs.datadoghq.com/tests/search/)
16
17
  - [Enhance developer workflows](https://docs.datadoghq.com/tests/developer_workflows)
17
18
  - [Flaky test management](https://docs.datadoghq.com/tests/guides/flaky_test_management/)
@@ -249,7 +249,7 @@ static int process_instantiated_klass(st_data_t key, st_data_t _value, st_data_t
249
249
  }
250
250
 
251
251
  VALUE filename = RARRAY_AREF(source_location, 0);
252
- if (filename == Qnil)
252
+ if (filename == Qnil || !RB_TYPE_P(filename, T_STRING))
253
253
  {
254
254
  return ST_CONTINUE;
255
255
  }
@@ -11,6 +11,7 @@ require_relative "../test_optimisation/coverage/transport"
11
11
  require_relative "../test_optimisation/coverage/writer"
12
12
  require_relative "../test_retries/component"
13
13
  require_relative "../test_retries/null_component"
14
+ require_relative "../test_retries/unique_tests_client"
14
15
  require_relative "../test_visibility/component"
15
16
  require_relative "../test_visibility/flush"
16
17
  require_relative "../test_visibility/null_component"
@@ -19,7 +20,6 @@ require_relative "../test_visibility/serializers/factories/test_suite_level"
19
20
  require_relative "../test_visibility/transport"
20
21
  require_relative "../transport/adapters/telemetry_webmock_safe_adapter"
21
22
  require_relative "../transport/api/builder"
22
- require_relative "../utils/identity"
23
23
  require_relative "../utils/parsing"
24
24
  require_relative "../utils/test_run"
25
25
  require_relative "../worker"
@@ -116,7 +116,9 @@ module Datadog
116
116
  @test_retries = TestRetries::Component.new(
117
117
  retry_failed_tests_enabled: settings.ci.retry_failed_tests_enabled,
118
118
  retry_failed_tests_max_attempts: settings.ci.retry_failed_tests_max_attempts,
119
- retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit
119
+ retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit,
120
+ retry_new_tests_enabled: settings.ci.retry_new_tests_enabled,
121
+ unique_tests_client: build_unique_tests_client(settings, test_visibility_api)
120
122
  )
121
123
  # @type ivar @test_optimisation: Datadog::CI::TestOptimisation::Component
122
124
  @test_optimisation = build_test_optimisation(settings, test_visibility_api)
@@ -236,6 +238,14 @@ module Datadog
236
238
  )
237
239
  end
238
240
 
241
+ def build_unique_tests_client(settings, api)
242
+ TestRetries::UniqueTestsClient.new(
243
+ api: api,
244
+ dd_env: settings.env,
245
+ config_tags: custom_configuration(settings)
246
+ )
247
+ end
248
+
239
249
  # fetch custom tags provided by the user in DD_TAGS env var
240
250
  # with prefix test.configuration.
241
251
  def custom_configuration(settings)
@@ -274,12 +284,8 @@ module Datadog
274
284
  settings.telemetry.shutdown_timeout_seconds = 60.0
275
285
 
276
286
  begin
277
- require "datadog/core/environment/identity"
278
287
  require "datadog/core/telemetry/http/adapters/net"
279
288
 
280
- # patch gem's identity to report datadog-ci library version instead of datadog gem version
281
- Core::Environment::Identity.include(CI::Utils::Identity)
282
-
283
289
  # patch gem's telemetry transport layer to use Net::HTTP instead of WebMock's Net::HTTP
284
290
  Core::Telemetry::Http::Adapters::Net.include(CI::Transport::Adapters::TelemetryWebmockSafeAdapter)
285
291
  rescue => e
@@ -106,6 +106,12 @@ module Datadog
106
106
  o.default 1000
107
107
  end
108
108
 
109
+ option :retry_new_tests_enabled do |o|
110
+ o.type :bool
111
+ o.env CI::Ext::Settings::ENV_RETRY_NEW_TESTS_ENABLED
112
+ o.default true
113
+ end
114
+
109
115
  define_method(:instrument) do |integration_name, options = {}, &block|
110
116
  return unless enabled
111
117
 
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ module CI
5
+ module Contrib
6
+ module Cucumber
7
+ class Filter < ::Cucumber::Core::Filter.new(:configuration)
8
+ def test_case(test_case)
9
+ test_retries_component.reset_retries! unless test_case_seen[test_case]
10
+ test_case_seen[test_case] = true
11
+
12
+ configuration.on_event(:test_case_finished) do |event|
13
+ next unless retry_required?(test_case, event)
14
+
15
+ test_case.describe_to(receiver)
16
+ end
17
+
18
+ super
19
+ end
20
+
21
+ private
22
+
23
+ def retry_required?(test_case, event)
24
+ return false unless event.test_case == test_case
25
+
26
+ test_retries_component.should_retry?
27
+ end
28
+
29
+ def test_case_seen
30
+ @test_case_seen ||= Hash.new { |h, k| h[k] = false }
31
+ end
32
+
33
+ def test_retries_component
34
+ @test_retries_component ||= Datadog.send(:components).test_retries
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
@@ -18,10 +18,24 @@ module Datadog
18
18
 
19
19
  def formatters
20
20
  existing_formatters = super
21
- @datadog_formatter ||= CI::Contrib::Cucumber::Formatter.new(@configuration)
21
+ @datadog_formatter ||= Formatter.new(@configuration)
22
22
  [@datadog_formatter] + existing_formatters
23
23
  end
24
24
 
25
+ def filters
26
+ require_relative "filter"
27
+
28
+ filters_list = super
29
+ datadog_filter = Filter.new(@configuration)
30
+ unless @configuration.dry_run?
31
+ # insert our filter the pre-last position because Cucumber::Filters::PrepareWorld must be the last one
32
+ # see:
33
+ # https://github.com/cucumber/cucumber-ruby/blob/58dd8f12c0ac5f4e607335ff2e7d385c1ed25899/lib/cucumber/runtime.rb#L266
34
+ filters_list.insert(-2, datadog_filter)
35
+ end
36
+ filters_list
37
+ end
38
+
25
39
  def begin_scenario(test_case)
26
40
  if Datadog::CI.active_test&.skipped_by_itr?
27
41
  raise ::Cucumber::Core::Test::Result::Skipped, CI::Ext::Test::ITR_TEST_SKIP_REASON
@@ -3,7 +3,6 @@
3
3
  require "datadog/tracing/contrib/patcher"
4
4
 
5
5
  require_relative "instrumentation"
6
- require_relative "configuration_override"
7
6
 
8
7
  module Datadog
9
8
  module CI
@@ -21,7 +20,6 @@ module Datadog
21
20
 
22
21
  def patch
23
22
  ::Cucumber::Runtime.include(Instrumentation)
24
- ::Cucumber::Configuration.include(ConfigurationOverride)
25
23
  end
26
24
  end
27
25
  end
@@ -8,6 +8,8 @@ module Datadog
8
8
  module Contrib
9
9
  module Minitest
10
10
  module Runner
11
+ DD_ESTIMATED_TESTS_PER_SUITE = 5
12
+
11
13
  def self.included(base)
12
14
  base.singleton_class.prepend(ClassMethods)
13
15
  end
@@ -18,12 +20,15 @@ module Datadog
18
20
 
19
21
  return unless datadog_configuration[:enabled]
20
22
 
23
+ # minitest does not store the total number of tests, so we can't pass it to the test session
24
+ # instead, we use the number of test suites * DD_ESTIMATED_TESTS_PER_SUITE as a rough estimate
21
25
  test_visibility_component.start_test_session(
22
26
  tags: {
23
27
  CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
24
28
  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s
25
29
  },
26
- service: datadog_configuration[:service_name]
30
+ service: datadog_configuration[:service_name],
31
+ total_tests_count: (DD_ESTIMATED_TESTS_PER_SUITE * ::Minitest::Runnable.runnables.size).to_i
27
32
  )
28
33
  test_visibility_component.start_test_module(Ext::FRAMEWORK)
29
34
  end
@@ -51,6 +51,10 @@ module Datadog
51
51
  return super unless test_span
52
52
 
53
53
  finish_with_result(test_span, result_code)
54
+
55
+ # remove failures if test passed at least once on retries
56
+ self.failures = [] if test_span.any_retry_passed?
57
+
54
58
  if Helpers.parallel?(self.class)
55
59
  finish_with_result(test_span.test_suite, result_code)
56
60
  end
@@ -64,9 +64,9 @@ module Datadog
64
64
 
65
65
  result = super
66
66
 
67
- # In case when test job is canceled and RSpec is quitting we don't want to report the last test
67
+ # When test job is canceled and RSpec is quitting we don't want to report the last test
68
68
  # before RSpec context unwinds. This test might have some unrelated errors that we don't want to
69
- # report.
69
+ # see in Datadog.
70
70
  return result if ::RSpec.world.wants_to_quit
71
71
 
72
72
  case execution_result.status
@@ -74,6 +74,8 @@ module Datadog
74
74
  test_span&.passed!
75
75
  when :failed
76
76
  test_span&.failed!(exception: execution_result.exception)
77
+ # if any of the retries passed, we don't fail the test run
78
+ @exception = nil if test_span&.any_retry_passed?
77
79
  else
78
80
  # :pending or nil
79
81
  test_span&.skipped!(
@@ -84,21 +86,20 @@ module Datadog
84
86
  end
85
87
  end
86
88
 
87
- # after retries are done, we can report the test to RSpec
88
- @skip_reporting = false
89
-
90
89
  # this is a special case for ci-queue, we need to finish the test suite span
91
90
  ci_queue_test_span&.finish
92
91
 
93
- # Finish spec with latest retry's result
94
- # TODO: when implementing new test retries make sure to clean @exception before calling this method
95
- # if test passed at least once
92
+ # after retries are done, we can finally report the test to RSpec
93
+ @skip_reporting = false
96
94
  finish(reporter)
97
95
  end
98
96
 
99
97
  def finish(reporter)
100
- # by default finish test but do not report it to RSpec::Core::Reporter
101
- # it is going to be reported once after retries are done
98
+ # By default finish test but do not report it to RSpec::Core::Reporter
99
+ # it is going to be reported once after retries are done.
100
+ #
101
+ # We need to do this because RSpec breaks when we try to report the same example multiple times with different
102
+ # results.
102
103
  return super unless @skip_reporting
103
104
 
104
105
  super(::RSpec::Core::NullReporter)
@@ -23,7 +23,8 @@ module Datadog
23
23
  CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
24
24
  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s
25
25
  },
26
- service: datadog_configuration[:service_name]
26
+ service: datadog_configuration[:service_name],
27
+ total_tests_count: ::RSpec.world.example_count
27
28
  )
28
29
 
29
30
  test_module = test_visibility_component.start_test_module(Ext::FRAMEWORK)
@@ -18,6 +18,7 @@ module Datadog
18
18
  ENV_RETRY_FAILED_TESTS_ENABLED = "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED"
19
19
  ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS = "DD_CIVISIBILITY_FLAKY_RETRY_COUNT"
20
20
  ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT = "DD_CIVISIBILITY_TOTAL_FLAKY_RETRY_COUNT"
21
+ ENV_RETRY_NEW_TESTS_ENABLED = "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED"
21
22
 
22
23
  # Source: https://docs.datadoghq.com/getting_started/site/
23
24
  DD_SITE_ALLOWLIST = %w[
@@ -54,6 +54,12 @@ module Datadog
54
54
  METRIC_CODE_COVERAGE_IS_EMPTY = "code_coverage.is_empty"
55
55
  METRIC_CODE_COVERAGE_FILES = "code_coverage.files"
56
56
 
57
+ METRIC_EFD_UNIQUE_TESTS_REQUEST = "early_flake_detection.request"
58
+ METRIC_EFD_UNIQUE_TESTS_REQUEST_MS = "early_flake_detection.request_ms"
59
+ METRIC_EFD_UNIQUE_TESTS_REQUEST_ERRORS = "early_flake_detection.request_errors"
60
+ METRIC_EFD_UNIQUE_TESTS_RESPONSE_BYTES = "early_flake_detection.response_bytes"
61
+ METRIC_EFD_UNIQUE_TESTS_RESPONSE_TESTS = "early_flake_detection.response_tests"
62
+
57
63
  METRIC_TEST_SESSION = "test_session"
58
64
 
59
65
  TAG_TEST_FRAMEWORK = "test_framework"
@@ -63,6 +69,7 @@ module Datadog
63
69
  TAG_BROWSER_DRIVER = "browser_driver"
64
70
  TAG_IS_RUM = "is_rum"
65
71
  TAG_IS_RETRY = "is_retry"
72
+ TAG_IS_NEW = "is_new"
66
73
  TAG_LIBRARY = "library"
67
74
  TAG_ENDPOINT = "endpoint"
68
75
  TAG_ERROR_TYPE = "error_type"
@@ -73,6 +80,8 @@ module Datadog
73
80
  TAG_COMMAND = "command"
74
81
  TAG_COVERAGE_ENABLED = "coverage_enabled"
75
82
  TAG_ITR_SKIP_ENABLED = "itrskip_enabled"
83
+ TAG_EARLY_FLAKE_DETECTION_ENABLED = "early_flake_detection_enabled"
84
+ TAG_EARLY_FLAKE_DETECTION_ABORT_REASON = "early_flake_detection_abort_reason"
76
85
  TAG_PROVIDER = "provider"
77
86
  TAG_AUTO_INJECTED = "auto_injected"
78
87
 
@@ -58,8 +58,11 @@ module Datadog
58
58
  # version of the browser, if multiple browsers or multiple versions then this tag is empty
59
59
  TAG_BROWSER_VERSION = "test.browser.version"
60
60
 
61
- # Tags for test retries
61
+ # Tags for retries
62
62
  TAG_IS_RETRY = "test.is_retry" # true if test was retried by datadog-ci library
63
+ TAG_IS_NEW = "test.is_new" # true if test was marked as new by new test retries (early flake detection)
64
+ TAG_EARLY_FLAKE_ENABLED = "test.early_flake.enabled" # true if early flake detection is enabled
65
+ TAG_EARLY_FLAKE_ABORT_REASON = "test.early_flake.abort_reason" # reason why early flake detection was aborted
63
66
 
64
67
  # internal APM tag to mark a span as a test span
65
68
  TAG_SPAN_KIND = "span.kind"
@@ -73,6 +76,8 @@ module Datadog
73
76
  ITR_TEST_SKIP_REASON = "Skipped by Datadog's intelligent test runner"
74
77
  ITR_UNSKIPPABLE_OPTION = :datadog_itr_unskippable
75
78
 
79
+ EARLY_FLAKE_FAULTY = "faulty"
80
+
76
81
  # test status as recognized by Datadog
77
82
  module Status
78
83
  PASS = "pass"
@@ -37,6 +37,10 @@ module Datadog
37
37
  DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY = "tests_skipping"
38
38
  DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY = "require_git"
39
39
  DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY = "flaky_test_retries_enabled"
40
+ DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY = "early_flake_detection"
41
+ DD_API_SETTINGS_RESPONSE_ENABLED_KEY = "enabled"
42
+ DD_API_SETTINGS_RESPONSE_SLOW_TEST_RETRIES_KEY = "slow_test_retries"
43
+ DD_API_SETTINGS_RESPONSE_FAULTY_SESSION_THRESHOLD_KEY = "faulty_session_threshold"
40
44
  DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze
41
45
 
42
46
  DD_API_GIT_SEARCH_COMMITS_PATH = "/api/v2/git/repository/search_commits"
@@ -46,6 +50,9 @@ module Datadog
46
50
  DD_API_SKIPPABLE_TESTS_PATH = "/api/v2/ci/tests/skippable"
47
51
  DD_API_SKIPPABLE_TESTS_TYPE = "test_params"
48
52
 
53
+ DD_API_UNIQUE_TESTS_PATH = "/api/v2/ci/libraries/tests"
54
+ DD_API_UNIQUE_TESTS_TYPE = "ci_app_libraries_tests_request"
55
+
49
56
  CONTENT_TYPE_MESSAGEPACK = "application/msgpack"
50
57
  CONTENT_TYPE_JSON = "application/json"
51
58
  CONTENT_TYPE_MULTIPART_FORM_DATA = "multipart/form-data"
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative "../worker"
4
+
3
5
  module Datadog
4
6
  module CI
5
7
  module Remote
@@ -27,8 +29,17 @@ module Datadog
27
29
  end
28
30
  end
29
31
 
30
- test_optimisation.configure(library_configuration, test_session)
31
- test_retries.configure(library_configuration)
32
+ # configure different components in parallel because they might block on HTTP requests
33
+ configuration_workers = [
34
+ Worker.new { test_optimisation.configure(library_configuration, test_session) },
35
+ Worker.new { test_retries.configure(library_configuration, test_session) }
36
+ ]
37
+
38
+ # launch configuration workers
39
+ configuration_workers.each(&:perform)
40
+
41
+ # block until all workers are done (or 60 seconds has passed)
42
+ configuration_workers.each(&:wait_until_done)
32
43
  end
33
44
 
34
45
  private
@@ -7,6 +7,8 @@ require_relative "../ext/transport"
7
7
  require_relative "../transport/telemetry"
8
8
  require_relative "../utils/parsing"
9
9
 
10
+ require_relative "slow_test_retries"
11
+
10
12
  module Datadog
11
13
  module CI
12
14
  module Remote
@@ -49,37 +51,76 @@ module Datadog
49
51
  def require_git?
50
52
  return @require_git if defined?(@require_git)
51
53
 
52
- @require_git = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY)
54
+ @require_git = Utils::Parsing.convert_to_bool(
55
+ payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY, false)
56
+ )
53
57
  end
54
58
 
55
59
  def itr_enabled?
56
60
  return @itr_enabled if defined?(@itr_enabled)
57
61
 
58
- @itr_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY)
62
+ @itr_enabled = Utils::Parsing.convert_to_bool(
63
+ payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY, false)
64
+ )
59
65
  end
60
66
 
61
67
  def code_coverage_enabled?
62
68
  return @code_coverage_enabled if defined?(@code_coverage_enabled)
63
69
 
64
- @code_coverage_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY)
70
+ @code_coverage_enabled = Utils::Parsing.convert_to_bool(
71
+ payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY, false)
72
+ )
65
73
  end
66
74
 
67
75
  def tests_skipping_enabled?
68
76
  return @tests_skipping_enabled if defined?(@tests_skipping_enabled)
69
77
 
70
- @tests_skipping_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY)
78
+ @tests_skipping_enabled = Utils::Parsing.convert_to_bool(
79
+ payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY, false)
80
+ )
71
81
  end
72
82
 
73
83
  def flaky_test_retries_enabled?
74
84
  return @flaky_test_retries_enabled if defined?(@flaky_test_retries_enabled)
75
85
 
76
- @flaky_test_retries_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY)
86
+ @flaky_test_retries_enabled = Utils::Parsing.convert_to_bool(
87
+ payload.fetch(
88
+ Ext::Transport::DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY, false
89
+ )
90
+ )
91
+ end
92
+
93
+ def early_flake_detection_enabled?
94
+ return @early_flake_detection_enabled if defined?(@early_flake_detection_enabled)
95
+
96
+ @early_flake_detection_enabled = Utils::Parsing.convert_to_bool(
97
+ early_flake_detection_payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_ENABLED_KEY, false)
98
+ )
99
+ end
100
+
101
+ def slow_test_retries
102
+ return @slow_test_retries if defined?(@slow_test_retries)
103
+
104
+ @slow_test_retries = SlowTestRetries.new(
105
+ early_flake_detection_payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_SLOW_TEST_RETRIES_KEY, {})
106
+ )
107
+ end
108
+
109
+ def faulty_session_threshold
110
+ return @faulty_session_threshold if defined?(@faulty_session_threshold)
111
+
112
+ @faulty_session_threshold = early_flake_detection_payload.fetch(
113
+ Ext::Transport::DD_API_SETTINGS_RESPONSE_FAULTY_SESSION_THRESHOLD_KEY, 0
114
+ )
77
115
  end
78
116
 
79
117
  private
80
118
 
81
- def bool(key)
82
- Utils::Parsing.convert_to_bool(payload.fetch(key, false))
119
+ def early_flake_detection_payload
120
+ payload.fetch(
121
+ Ext::Transport::DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY,
122
+ {}
123
+ )
83
124
  end
84
125
 
85
126
  def default_payload
@@ -58,7 +58,8 @@ module Datadog
58
58
  1,
59
59
  {
60
60
  Ext::Telemetry::TAG_COVERAGE_ENABLED => library_settings.code_coverage_enabled?.to_s,
61
- Ext::Telemetry::TAG_ITR_SKIP_ENABLED => library_settings.tests_skipping_enabled?.to_s
61
+ Ext::Telemetry::TAG_ITR_SKIP_ENABLED => library_settings.tests_skipping_enabled?.to_s,
62
+ Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ENABLED => library_settings.early_flake_detection_enabled?.to_s
62
63
  }
63
64
  )
64
65
 
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ module CI
5
+ module Remote
6
+ # Parses "slow_test_retries" payload for early flake detection settings
7
+ #
8
+ # Example payload:
9
+ # {
10
+ # "5s" => 10,
11
+ # "10s" => 5,
12
+ # "30s" => 3,
13
+ # "5m" => 2
14
+ # }
15
+ #
16
+ # The payload above means that for tests that run less than 5 seconds, we should retry them 10 times,
17
+ # for tests that run less than 10 seconds, we should retry them 5 times, and so on.
18
+ class SlowTestRetries
19
+ attr_reader :entries
20
+
21
+ Entry = Struct.new(:duration, :max_attempts)
22
+
23
+ DURATION_MEASURES = {
24
+ "s" => 1,
25
+ "m" => 60
26
+ }.freeze
27
+
28
+ def initialize(payload)
29
+ @entries = parse(payload)
30
+ end
31
+
32
+ def max_attempts_for_duration(duration)
33
+ @entries.each do |entry|
34
+ return entry.max_attempts if duration < entry.duration
35
+ end
36
+
37
+ 0
38
+ end
39
+
40
+ private
41
+
42
+ def parse(payload)
43
+ (payload || {}).keys.filter_map do |key|
44
+ duration, measure = key.match(/(\d+)(\w+)/)&.captures
45
+ next if duration.nil? || measure.nil? || !DURATION_MEASURES.key?(measure)
46
+
47
+ Entry.new(duration.to_f * DURATION_MEASURES.fetch(measure, 1), payload[key].to_i)
48
+ end.sort_by(&:duration)
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
@@ -70,6 +70,12 @@ module Datadog
70
70
  get_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR) == "true"
71
71
  end
72
72
 
73
+ # Returns "true" if test span represents a retry.
74
+ # @return [Boolean] true if this test is a retry, false otherwise.
75
+ def is_retry?
76
+ get_tag(Ext::Test::TAG_IS_RETRY) == "true"
77
+ end
78
+
73
79
  # Marks this test as unskippable by the intelligent test runner.
74
80
  # This must be done before the test execution starts.
75
81
  #
@@ -139,11 +145,18 @@ module Datadog
139
145
  get_tag(Ext::Test::TAG_PARAMETERS)
140
146
  end
141
147
 
148
+ # @internal
149
+ def any_retry_passed?
150
+ !!test_suite&.any_test_retry_passed?(test_id)
151
+ end
152
+
142
153
  private
143
154
 
144
- def record_test_result(datadog_status)
145
- test_id = Utils::TestRun.skippable_test_id(name, test_suite_name, parameters)
155
+ def test_id
156
+ @test_id ||= Utils::TestRun.datadog_test_id(name, test_suite_name, parameters)
157
+ end
146
158
 
159
+ def record_test_result(datadog_status)
147
160
  # if this test was already executed in this test suite, mark it as retried
148
161
  if test_suite&.test_executed?(test_id)
149
162
  set_tag(Ext::Test::TAG_IS_RETRY, "true")