datadog-ci 1.4.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ddc261b14aac25efcc4c94bf30e18962e714a83e35d47cb3a1f3210d640fd5a
-  data.tar.gz: 47acdc6af95b2ab2e7385a22d0c82359e63ff0eabb8f0dfdf1bd0197f73eb6ee
+  metadata.gz: 876a0e7ada3aeaf87b19dade6ee73fec0fe8d37387cde5fc31b43aaad76e99c5
+  data.tar.gz: 8c46ad7c35a621ae03b7b42faa5b3c6934066e422ff4d3a8ac2249cd6466ecd3
 SHA512:
-  metadata.gz: 1a774a633e3da98fc313dcf2bfeb14c02e231fc52fc7abced54ae4246afd9930fe4598d5885e39a999ede252da6c9aa0262e041aad7b4873d4fcdb8349dab673
-  data.tar.gz: ed05b4354183b31a619f9f18aa22c547dc0ccbfdd829e8fa24e250fe98becd2a434855bb71839325214d017d6e1b79c99dcf681eeb7f820e1508991516722070
+  metadata.gz: 8f312e6920cad6072665c00dc5d253e66a7c3272abda20e79f86cf11beb38fffe0be789f644affb6f8920e458f00f221c7969fb84c2ee6f2198af6a140eafc64
+  data.tar.gz: f9b7c2b184e7a140269eaa2069cbf5062270d6bb07410fd9b46cab4c521a8d0bd694a676086164c58300c264357e53e0118611053f6bdf0c2dee757b487b4769

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
 
+## [1.5.0] - 2024-09-18
+
+### Added
+* Retry new tests - parse remote configuration and fetch unique known tests ([#227][])
+* early flake detection support for rspec and minitest ([#229][])
+* Early flake detection support for Cucumber ([#231][])
+
+### Fixed
+* Minor telemetry fixes ([#226][])
+
 ## [1.4.1] - 2024-08-28
 
 ### Fixed
@@ -315,7 +325,8 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 
 - Ruby versions < 2.7 no longer supported ([#8][])
 
-[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.4.1...main
+[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.5.0...main
+[1.5.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.4.1...v1.5.0
 [1.4.1]: https://github.com/DataDog/datadog-ci-rb/compare/v1.4.0...v1.4.1
 [1.4.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.3.0...v1.4.0
 [1.3.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.2.0...v1.3.0
@@ -452,4 +463,8 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 [#219]: https://github.com/DataDog/datadog-ci-rb/issues/219
 [#220]: https://github.com/DataDog/datadog-ci-rb/issues/220
 [#221]: https://github.com/DataDog/datadog-ci-rb/issues/221
-[#224]: https://github.com/DataDog/datadog-ci-rb/issues/224
+[#224]: https://github.com/DataDog/datadog-ci-rb/issues/224
+[#226]: https://github.com/DataDog/datadog-ci-rb/issues/226
+[#227]: https://github.com/DataDog/datadog-ci-rb/issues/227
+[#229]: https://github.com/DataDog/datadog-ci-rb/issues/229
+[#231]: https://github.com/DataDog/datadog-ci-rb/issues/231

data/lib/datadog/ci/configuration/components.rb

@@ -11,6 +11,7 @@ require_relative "../test_optimisation/coverage/transport"
 require_relative "../test_optimisation/coverage/writer"
 require_relative "../test_retries/component"
 require_relative "../test_retries/null_component"
+require_relative "../test_retries/unique_tests_client"
 require_relative "../test_visibility/component"
 require_relative "../test_visibility/flush"
 require_relative "../test_visibility/null_component"
@@ -19,7 +20,6 @@ require_relative "../test_visibility/serializers/factories/test_suite_level"
 require_relative "../test_visibility/transport"
 require_relative "../transport/adapters/telemetry_webmock_safe_adapter"
 require_relative "../transport/api/builder"
-require_relative "../utils/identity"
 require_relative "../utils/parsing"
 require_relative "../utils/test_run"
 require_relative "../worker"
@@ -116,7 +116,9 @@ module Datadog
           @test_retries = TestRetries::Component.new(
             retry_failed_tests_enabled: settings.ci.retry_failed_tests_enabled,
             retry_failed_tests_max_attempts: settings.ci.retry_failed_tests_max_attempts,
-            retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit
+            retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit,
+            retry_new_tests_enabled: settings.ci.retry_new_tests_enabled,
+            unique_tests_client: build_unique_tests_client(settings, test_visibility_api)
           )
           # @type ivar @test_optimisation: Datadog::CI::TestOptimisation::Component
           @test_optimisation = build_test_optimisation(settings, test_visibility_api)
@@ -236,6 +238,14 @@ module Datadog
           )
         end
 
+        def build_unique_tests_client(settings, api)
+          TestRetries::UniqueTestsClient.new(
+            api: api,
+            dd_env: settings.env,
+            config_tags: custom_configuration(settings)
+          )
+        end
+
         # fetch custom tags provided by the user in DD_TAGS env var
         # with prefix test.configuration.
         def custom_configuration(settings)
@@ -274,12 +284,8 @@ module Datadog
           settings.telemetry.shutdown_timeout_seconds = 60.0
 
           begin
-            require "datadog/core/environment/identity"
             require "datadog/core/telemetry/http/adapters/net"
 
-            # patch gem's identity to report datadog-ci library version instead of datadog gem version
-            Core::Environment::Identity.include(CI::Utils::Identity)
-
             # patch gem's telemetry transport layer to use Net::HTTP instead of WebMock's Net::HTTP
             Core::Telemetry::Http::Adapters::Net.include(CI::Transport::Adapters::TelemetryWebmockSafeAdapter)
           rescue => e

data/lib/datadog/ci/configuration/settings.rb

@@ -106,6 +106,12 @@ module Datadog
                 o.default 1000
               end
 
+              option :retry_new_tests_enabled do |o|
+                o.type :bool
+                o.env CI::Ext::Settings::ENV_RETRY_NEW_TESTS_ENABLED
+                o.default true
+              end
+
               define_method(:instrument) do |integration_name, options = {}, &block|
                 return unless enabled
 

data/lib/datadog/ci/contrib/cucumber/filter.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module Cucumber
+        class Filter < ::Cucumber::Core::Filter.new(:configuration)
+          def test_case(test_case)
+            test_retries_component.reset_retries! unless test_case_seen[test_case]
+            test_case_seen[test_case] = true
+
+            configuration.on_event(:test_case_finished) do |event|
+              next unless retry_required?(test_case, event)
+
+              test_case.describe_to(receiver)
+            end
+
+            super
+          end
+
+          private
+
+          def retry_required?(test_case, event)
+            return false unless event.test_case == test_case
+
+            test_retries_component.should_retry?
+          end
+
+          def test_case_seen
+            @test_case_seen ||= Hash.new { |h, k| h[k] = false }
+          end
+
+          def test_retries_component
+            @test_retries_component ||= Datadog.send(:components).test_retries
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/cucumber/instrumentation.rb

@@ -18,10 +18,24 @@ module Datadog
 
             def formatters
               existing_formatters = super
-              @datadog_formatter ||= CI::Contrib::Cucumber::Formatter.new(@configuration)
+              @datadog_formatter ||= Formatter.new(@configuration)
               [@datadog_formatter] + existing_formatters
             end
 
+            def filters
+              require_relative "filter"
+
+              filters_list = super
+              datadog_filter = Filter.new(@configuration)
+              unless @configuration.dry_run?
+                # insert our filter the pre-last position because Cucumber::Filters::PrepareWorld must be the last one
+                # see:
+                # https://github.com/cucumber/cucumber-ruby/blob/58dd8f12c0ac5f4e607335ff2e7d385c1ed25899/lib/cucumber/runtime.rb#L266
+                filters_list.insert(-2, datadog_filter)
+              end
+              filters_list
+            end
+
             def begin_scenario(test_case)
               if Datadog::CI.active_test&.skipped_by_itr?
                 raise ::Cucumber::Core::Test::Result::Skipped, CI::Ext::Test::ITR_TEST_SKIP_REASON

data/lib/datadog/ci/contrib/cucumber/patcher.rb

@@ -3,7 +3,6 @@
 require "datadog/tracing/contrib/patcher"
 
 require_relative "instrumentation"
-require_relative "configuration_override"
 
 module Datadog
   module CI
@@ -21,7 +20,6 @@ module Datadog
 
           def patch
             ::Cucumber::Runtime.include(Instrumentation)
-            ::Cucumber::Configuration.include(ConfigurationOverride)
           end
         end
       end

data/lib/datadog/ci/contrib/minitest/runner.rb

@@ -8,6 +8,8 @@ module Datadog
     module Contrib
       module Minitest
         module Runner
+          DD_ESTIMATED_TESTS_PER_SUITE = 5
+
           def self.included(base)
             base.singleton_class.prepend(ClassMethods)
           end
@@ -18,12 +20,15 @@ module Datadog
 
               return unless datadog_configuration[:enabled]
 
+              # minitest does not store the total number of tests, so we can't pass it to the test session
+              # instead, we use the number of test suites * DD_ESTIMATED_TESTS_PER_SUITE as a rough estimate
               test_visibility_component.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s
                 },
-                service: datadog_configuration[:service_name]
+                service: datadog_configuration[:service_name],
+                total_tests_count: (DD_ESTIMATED_TESTS_PER_SUITE * ::Minitest::Runnable.runnables.size).to_i
               )
               test_visibility_component.start_test_module(Ext::FRAMEWORK)
             end

data/lib/datadog/ci/contrib/minitest/test.rb

@@ -51,6 +51,10 @@ module Datadog
               return super unless test_span
 
               finish_with_result(test_span, result_code)
+
+              # remove failures if test passed at least once on retries
+              self.failures = [] if test_span.any_retry_passed?
+
               if Helpers.parallel?(self.class)
                 finish_with_result(test_span.test_suite, result_code)
               end

data/lib/datadog/ci/contrib/rspec/example.rb

@@ -64,9 +64,9 @@ module Datadog
 
                   result = super
 
-                  # In case when test job is canceled and RSpec is quitting we don't want to report the last test
+                  # When test job is canceled and RSpec is quitting we don't want to report the last test
                   # before RSpec context unwinds. This test might have some unrelated errors that we don't want to
-                  # report.
+                  # see in Datadog.
                   return result if ::RSpec.world.wants_to_quit
 
                   case execution_result.status
@@ -74,6 +74,8 @@ module Datadog
                     test_span&.passed!
                   when :failed
                     test_span&.failed!(exception: execution_result.exception)
+                    # if any of the retries passed, we don't fail the test run
+                    @exception = nil if test_span&.any_retry_passed?
                   else
                     # :pending or nil
                     test_span&.skipped!(
@@ -84,21 +86,20 @@ module Datadog
                 end
               end
 
-              # after retries are done, we can report the test to RSpec
-              @skip_reporting = false
-
               # this is a special case for ci-queue, we need to finish the test suite span
               ci_queue_test_span&.finish
 
-              # Finish spec with latest retry's result
-              # TODO: when implementing new test retries make sure to clean @exception before calling this method
-              # if test passed at least once
+              # after retries are done, we can finally report the test to RSpec
+              @skip_reporting = false
               finish(reporter)
             end
 
             def finish(reporter)
-              # by default finish test but do not report it to RSpec::Core::Reporter
-              # it is going to be reported once after retries are done
+              # By default finish test but do not report it to RSpec::Core::Reporter
+              # it is going to be reported once after retries are done.
+              #
+              # We need to do this because RSpec breaks when we try to report the same example multiple times with different
+              # results.
               return super unless @skip_reporting
 
               super(::RSpec::Core::NullReporter)

data/lib/datadog/ci/contrib/rspec/runner.rb

@@ -23,7 +23,8 @@ module Datadog
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s
                 },
-                service: datadog_configuration[:service_name]
+                service: datadog_configuration[:service_name],
+                total_tests_count: ::RSpec.world.example_count
               )
 
               test_module = test_visibility_component.start_test_module(Ext::FRAMEWORK)

data/lib/datadog/ci/ext/settings.rb

@@ -18,6 +18,7 @@ module Datadog
         ENV_RETRY_FAILED_TESTS_ENABLED = "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED"
         ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS = "DD_CIVISIBILITY_FLAKY_RETRY_COUNT"
         ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT = "DD_CIVISIBILITY_TOTAL_FLAKY_RETRY_COUNT"
+        ENV_RETRY_NEW_TESTS_ENABLED = "DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = %w[

data/lib/datadog/ci/ext/telemetry.rb

@@ -54,6 +54,12 @@ module Datadog
         METRIC_CODE_COVERAGE_IS_EMPTY = "code_coverage.is_empty"
         METRIC_CODE_COVERAGE_FILES = "code_coverage.files"
 
+        METRIC_EFD_UNIQUE_TESTS_REQUEST = "early_flake_detection.request"
+        METRIC_EFD_UNIQUE_TESTS_REQUEST_MS = "early_flake_detection.request_ms"
+        METRIC_EFD_UNIQUE_TESTS_REQUEST_ERRORS = "early_flake_detection.request_errors"
+        METRIC_EFD_UNIQUE_TESTS_RESPONSE_BYTES = "early_flake_detection.response_bytes"
+        METRIC_EFD_UNIQUE_TESTS_RESPONSE_TESTS = "early_flake_detection.response_tests"
+
         METRIC_TEST_SESSION = "test_session"
 
         TAG_TEST_FRAMEWORK = "test_framework"
@@ -63,6 +69,7 @@ module Datadog
         TAG_BROWSER_DRIVER = "browser_driver"
         TAG_IS_RUM = "is_rum"
         TAG_IS_RETRY = "is_retry"
+        TAG_IS_NEW = "is_new"
         TAG_LIBRARY = "library"
         TAG_ENDPOINT = "endpoint"
         TAG_ERROR_TYPE = "error_type"
@@ -73,6 +80,8 @@ module Datadog
         TAG_COMMAND = "command"
         TAG_COVERAGE_ENABLED = "coverage_enabled"
         TAG_ITR_SKIP_ENABLED = "itrskip_enabled"
+        TAG_EARLY_FLAKE_DETECTION_ENABLED = "early_flake_detection_enabled"
+        TAG_EARLY_FLAKE_DETECTION_ABORT_REASON = "early_flake_detection_abort_reason"
         TAG_PROVIDER = "provider"
         TAG_AUTO_INJECTED = "auto_injected"
 

data/lib/datadog/ci/ext/test.rb

@@ -58,8 +58,11 @@ module Datadog
         # version of the browser, if multiple browsers or multiple versions then this tag is empty
         TAG_BROWSER_VERSION = "test.browser.version"
 
-        # Tags for test retries
+        # Tags for retries
         TAG_IS_RETRY = "test.is_retry" # true if test was retried by datadog-ci library
+        TAG_IS_NEW = "test.is_new" # true if test was marked as new by new test retries (early flake detection)
+        TAG_EARLY_FLAKE_ENABLED = "test.early_flake.enabled" # true if early flake detection is enabled
+        TAG_EARLY_FLAKE_ABORT_REASON = "test.early_flake.abort_reason" # reason why early flake detection was aborted
 
         # internal APM tag to mark a span as a test span
         TAG_SPAN_KIND = "span.kind"
@@ -73,6 +76,8 @@ module Datadog
         ITR_TEST_SKIP_REASON = "Skipped by Datadog's intelligent test runner"
         ITR_UNSKIPPABLE_OPTION = :datadog_itr_unskippable
 
+        EARLY_FLAKE_FAULTY = "faulty"
+
         # test status as recognized by Datadog
         module Status
           PASS = "pass"

data/lib/datadog/ci/ext/transport.rb

@@ -37,6 +37,10 @@ module Datadog
         DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY = "tests_skipping"
         DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY = "require_git"
         DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY = "flaky_test_retries_enabled"
+        DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY = "early_flake_detection"
+        DD_API_SETTINGS_RESPONSE_ENABLED_KEY = "enabled"
+        DD_API_SETTINGS_RESPONSE_SLOW_TEST_RETRIES_KEY = "slow_test_retries"
+        DD_API_SETTINGS_RESPONSE_FAULTY_SESSION_THRESHOLD_KEY = "faulty_session_threshold"
         DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze
 
         DD_API_GIT_SEARCH_COMMITS_PATH = "/api/v2/git/repository/search_commits"
@@ -46,6 +50,9 @@ module Datadog
         DD_API_SKIPPABLE_TESTS_PATH = "/api/v2/ci/tests/skippable"
         DD_API_SKIPPABLE_TESTS_TYPE = "test_params"
 
+        DD_API_UNIQUE_TESTS_PATH = "/api/v2/ci/libraries/tests"
+        DD_API_UNIQUE_TESTS_TYPE = "ci_app_libraries_tests_request"
+
         CONTENT_TYPE_MESSAGEPACK = "application/msgpack"
         CONTENT_TYPE_JSON = "application/json"
         CONTENT_TYPE_MULTIPART_FORM_DATA = "multipart/form-data"

data/lib/datadog/ci/remote/component.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../worker"
+
 module Datadog
   module CI
     module Remote
@@ -27,8 +29,17 @@ module Datadog
             end
           end
 
-          test_optimisation.configure(library_configuration, test_session)
-          test_retries.configure(library_configuration)
+          # configure different components in parallel because they might block on HTTP requests
+          configuration_workers = [
+            Worker.new { test_optimisation.configure(library_configuration, test_session) },
+            Worker.new { test_retries.configure(library_configuration, test_session) }
+          ]
+
+          # launch configuration workers
+          configuration_workers.each(&:perform)
+
+          # block until all workers are done (or 60 seconds has passed)
+          configuration_workers.each(&:wait_until_done)
         end
 
         private

data/lib/datadog/ci/remote/library_settings.rb

@@ -7,6 +7,8 @@ require_relative "../ext/transport"
 require_relative "../transport/telemetry"
 require_relative "../utils/parsing"
 
+require_relative "slow_test_retries"
+
 module Datadog
   module CI
     module Remote
@@ -49,37 +51,76 @@ module Datadog
         def require_git?
           return @require_git if defined?(@require_git)
 
-          @require_git = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY)
+          @require_git = Utils::Parsing.convert_to_bool(
+            payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY, false)
+          )
         end
 
         def itr_enabled?
           return @itr_enabled if defined?(@itr_enabled)
 
-          @itr_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY)
+          @itr_enabled = Utils::Parsing.convert_to_bool(
+            payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY, false)
+          )
         end
 
         def code_coverage_enabled?
           return @code_coverage_enabled if defined?(@code_coverage_enabled)
 
-          @code_coverage_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY)
+          @code_coverage_enabled = Utils::Parsing.convert_to_bool(
+            payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY, false)
+          )
         end
 
         def tests_skipping_enabled?
           return @tests_skipping_enabled if defined?(@tests_skipping_enabled)
 
-          @tests_skipping_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY)
+          @tests_skipping_enabled = Utils::Parsing.convert_to_bool(
+            payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY, false)
+          )
         end
 
         def flaky_test_retries_enabled?
           return @flaky_test_retries_enabled if defined?(@flaky_test_retries_enabled)
 
-          @flaky_test_retries_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY)
+          @flaky_test_retries_enabled = Utils::Parsing.convert_to_bool(
+            payload.fetch(
+              Ext::Transport::DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY, false
+            )
+          )
+        end
+
+        def early_flake_detection_enabled?
+          return @early_flake_detection_enabled if defined?(@early_flake_detection_enabled)
+
+          @early_flake_detection_enabled = Utils::Parsing.convert_to_bool(
+            early_flake_detection_payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_ENABLED_KEY, false)
+          )
+        end
+
+        def slow_test_retries
+          return @slow_test_retries if defined?(@slow_test_retries)
+
+          @slow_test_retries = SlowTestRetries.new(
+            early_flake_detection_payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_SLOW_TEST_RETRIES_KEY, {})
+          )
+        end
+
+        def faulty_session_threshold
+          return @faulty_session_threshold if defined?(@faulty_session_threshold)
+
+          @faulty_session_threshold = early_flake_detection_payload.fetch(
+            Ext::Transport::DD_API_SETTINGS_RESPONSE_FAULTY_SESSION_THRESHOLD_KEY, 0
+          )
         end
 
         private
 
-        def bool(key)
-          Utils::Parsing.convert_to_bool(payload.fetch(key, false))
+        def early_flake_detection_payload
+          payload.fetch(
+            Ext::Transport::DD_API_SETTINGS_RESPONSE_EARLY_FLAKE_DETECTION_KEY,
+            {}
+          )
         end
 
         def default_payload

data/lib/datadog/ci/remote/library_settings_client.rb

@@ -58,7 +58,8 @@ module Datadog
             1,
             {
               Ext::Telemetry::TAG_COVERAGE_ENABLED => library_settings.code_coverage_enabled?.to_s,
-              Ext::Telemetry::TAG_ITR_SKIP_ENABLED => library_settings.tests_skipping_enabled?.to_s
+              Ext::Telemetry::TAG_ITR_SKIP_ENABLED => library_settings.tests_skipping_enabled?.to_s,
+              Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ENABLED => library_settings.early_flake_detection_enabled?.to_s
             }
           )
 

data/lib/datadog/ci/remote/slow_test_retries.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Remote
+      # Parses "slow_test_retries" payload for early flake detection settings
+      #
+      # Example payload:
+      # {
+      #  "5s" => 10,
+      #  "10s" => 5,
+      #  "30s" => 3,
+      #  "5m" => 2
+      # }
+      #
+      # The payload above means that for tests that run less than 5 seconds, we should retry them 10 times,
+      # for tests that run less than 10 seconds, we should retry them 5 times, and so on.
+      class SlowTestRetries
+        attr_reader :entries
+
+        Entry = Struct.new(:duration, :max_attempts)
+
+        DURATION_MEASURES = {
+          "s" => 1,
+          "m" => 60
+        }.freeze
+
+        def initialize(payload)
+          @entries = parse(payload)
+        end
+
+        def max_attempts_for_duration(duration)
+          @entries.each do |entry|
+            return entry.max_attempts if duration < entry.duration
+          end
+
+          0
+        end
+
+        private
+
+        def parse(payload)
+          (payload || {}).keys.filter_map do |key|
+            duration, measure = key.match(/(\d+)(\w+)/)&.captures
+            next if duration.nil? || measure.nil? || !DURATION_MEASURES.key?(measure)
+
+            Entry.new(duration.to_f * DURATION_MEASURES.fetch(measure, 1), payload[key].to_i)
+          end.sort_by(&:duration)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test.rb

@@ -70,6 +70,12 @@ module Datadog
         get_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR) == "true"
       end
 
+      # Returns "true" if test span represents a retry.
+      # @return [Boolean] true if this test is a retry, false otherwise.
+      def is_retry?
+        get_tag(Ext::Test::TAG_IS_RETRY) == "true"
+      end
+
       # Marks this test as unskippable by the intelligent test runner.
       # This must be done before the test execution starts.
       #
@@ -139,11 +145,18 @@ module Datadog
         get_tag(Ext::Test::TAG_PARAMETERS)
       end
 
+      # @internal
+      def any_retry_passed?
+        !!test_suite&.any_test_retry_passed?(test_id)
+      end
+
       private
 
-      def record_test_result(datadog_status)
-        test_id = Utils::TestRun.skippable_test_id(name, test_suite_name, parameters)
+      def test_id
+        @test_id ||= Utils::TestRun.datadog_test_id(name, test_suite_name, parameters)
+      end
 
+      def record_test_result(datadog_status)
         # if this test was already executed in this test suite, mark it as retried
         if test_suite&.test_executed?(test_id)
           set_tag(Ext::Test::TAG_IS_RETRY, "true")

data/lib/datadog/ci/test_optimisation/component.rb

@@ -99,6 +99,7 @@ module Datadog
 
         def start_coverage(test)
           return if !enabled? || !code_coverage?
+
           Telemetry.code_coverage_started(test)
           coverage_collector&.start
         end
@@ -109,13 +110,15 @@ module Datadog
           Telemetry.code_coverage_finished(test)
 
           coverage = coverage_collector&.stop
+
+          # if test was skipped, we discard coverage data
+          return if test.skipped?
+
           if coverage.nil? || coverage.empty?
             Telemetry.code_coverage_is_empty
             return
           end
 
-          return if test.skipped?
-
           test_source_file = test.source_file
 
           # cucumber's gherkin files are not covered by the code coverage collector
@@ -140,8 +143,8 @@ module Datadog
         def mark_if_skippable(test)
           return if !enabled? || !skipping_tests?
 
-          skippable_test_id = Utils::TestRun.skippable_test_id(test.name, test.test_suite_name, test.parameters)
-          if @skippable_tests.include?(skippable_test_id)
+          datadog_test_id = Utils::TestRun.datadog_test_id(test.name, test.test_suite_name, test.parameters)
+          if @skippable_tests.include?(datadog_test_id)
             if forked?
               Datadog.logger.warn { "Intelligent test runner is not supported for forking test runners yet" }
               return
@@ -149,9 +152,9 @@ module Datadog
 
             test.set_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true")
 
-            Datadog.logger.debug { "Marked test as skippable: #{skippable_test_id}" }
+            Datadog.logger.debug { "Marked test as skippable: #{datadog_test_id}" }
           else
-            Datadog.logger.debug { "Test is not skippable: #{skippable_test_id}" }
+            Datadog.logger.debug { "Test is not skippable: #{datadog_test_id}" }
           end
         end
 

data/lib/datadog/ci/test_optimisation/skippable.rb

@@ -36,7 +36,7 @@ module Datadog
                 next unless test_data["type"] == Ext::Test::ITR_TEST_SKIPPING_MODE
 
                 attrs = test_data["attributes"] || {}
-                res << Utils::TestRun.skippable_test_id(attrs["name"], attrs["suite"], attrs["parameters"])
+                res << Utils::TestRun.datadog_test_id(attrs["name"], attrs["suite"], attrs["parameters"])
               end
 
             res