datadog-ci 1.3.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57da167788887f67dc501a3dc2869c425754fec3cdefa45770af22364003fb20
-  data.tar.gz: ed6645a822db10fbae793c5b226b9d41ca4dbea67449f545135f2571be3894c1
+  metadata.gz: 0ddc261b14aac25efcc4c94bf30e18962e714a83e35d47cb3a1f3210d640fd5a
+  data.tar.gz: 47acdc6af95b2ab2e7385a22d0c82359e63ff0eabb8f0dfdf1bd0197f73eb6ee
 SHA512:
-  metadata.gz: f4586bbf8a10b5bf2b262bafefd222ef981624ced261755bd9d43652506400f6b939e5cd3595b7af0c66a193581328937aabe5b9246efd4cb3a0fa088b9e0ccf
-  data.tar.gz: 4169cf67306d9d080e264897ed2cffb5103a9b98ca176fea2784ff3293d4b9dd3decd62df3848b2a7ade783f5dfe6925b86fa7d4727b45de51764af5bc6e1b4a
+  metadata.gz: 1a774a633e3da98fc313dcf2bfeb14c02e231fc52fc7abced54ae4246afd9930fe4598d5885e39a999ede252da6c9aa0262e041aad7b4873d4fcdb8349dab673
+  data.tar.gz: ed05b4354183b31a619f9f18aa22c547dc0ccbfdd829e8fa24e250fe98becd2a434855bb71839325214d017d6e1b79c99dcf681eeb7f820e1508991516722070

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 ## [Unreleased]
 
+## [1.4.1] - 2024-08-28
+
+### Fixed
+
+* fix datadog_cov crash when doing allocation profiling ([#224][])
+
+## [1.4.0] - 2024-08-26
+
+### Added
+
+* Auto test retries for cucumber ([#212][])
+* Auto test retries for RSpec ([#213][])
+* Auto test retries for minitest ([#214][])
+* implement auto test retries RFC ([#219][])
+
+### Changed
+
+* Skip Before/After hooks in cucumber when scenario is skipped by intelligent test runner ([#211][])
+* gem datadog 2.3 is now minimal required version ([#220][])
+* Enable agentless telemetry when library is running in agentless mode ([#221][])
+* Add Ruby 3.4 to the testing matrix ([#217][])
+* add different fallbacks for unshallowing remotes ([#218][])
+* make itr_enabled config parameter true by default ([#216][])
+* RSpec - don't report test errors if rspec process is quitting ([#215][])
+
 ## [1.3.0] - 2024-07-30
 
 ### Added
@@ -290,7 +315,9 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 
 - Ruby versions < 2.7 no longer supported ([#8][])
 
-[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.3.0...main
+[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.4.1...main
+[1.4.1]: https://github.com/DataDog/datadog-ci-rb/compare/v1.4.0...v1.4.1
+[1.4.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.3.0...v1.4.0
 [1.3.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.2.0...v1.3.0
 [1.2.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.1.0...v1.2.0
 [1.1.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.0.1...v1.1.0
@@ -413,4 +440,16 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 [#204]: https://github.com/DataDog/datadog-ci-rb/issues/204
 [#205]: https://github.com/DataDog/datadog-ci-rb/issues/205
 [#206]: https://github.com/DataDog/datadog-ci-rb/issues/206
-[#207]: https://github.com/DataDog/datadog-ci-rb/issues/207
+[#207]: https://github.com/DataDog/datadog-ci-rb/issues/207
+[#211]: https://github.com/DataDog/datadog-ci-rb/issues/211
+[#212]: https://github.com/DataDog/datadog-ci-rb/issues/212
+[#213]: https://github.com/DataDog/datadog-ci-rb/issues/213
+[#214]: https://github.com/DataDog/datadog-ci-rb/issues/214
+[#215]: https://github.com/DataDog/datadog-ci-rb/issues/215
+[#216]: https://github.com/DataDog/datadog-ci-rb/issues/216
+[#217]: https://github.com/DataDog/datadog-ci-rb/issues/217
+[#218]: https://github.com/DataDog/datadog-ci-rb/issues/218
+[#219]: https://github.com/DataDog/datadog-ci-rb/issues/219
+[#220]: https://github.com/DataDog/datadog-ci-rb/issues/220
+[#221]: https://github.com/DataDog/datadog-ci-rb/issues/221
+[#224]: https://github.com/DataDog/datadog-ci-rb/issues/224

data/README.md

@@ -12,6 +12,7 @@ Learn more on our [official website](https://docs.datadoghq.com/tests/) and chec
 
 - [Test Visibility](https://docs.datadoghq.com/tests/) - collect metrics and results for your tests
 - [Intelligent test runner](https://docs.datadoghq.com/intelligent_test_runner/) - save time by selectively running only tests affected by code changes
+- [Auto test retries](https://docs.datadoghq.com/tests/auto_test_retries/?tab=ruby) - retrying failing tests up to N times to avoid failing your build due to flaky tests
 - [Search and manage CI tests](https://docs.datadoghq.com/tests/search/)
 - [Enhance developer workflows](https://docs.datadoghq.com/tests/developer_workflows)
 - [Flaky test management](https://docs.datadoghq.com/tests/guides/flaky_test_management/)

data/ext/datadog_cov/datadog_cov.c

@@ -249,7 +249,7 @@ static int process_instantiated_klass(st_data_t key, st_data_t _value, st_data_t
   }
 
   VALUE filename = RARRAY_AREF(source_location, 0);
-  if (filename == Qnil)
+  if (filename == Qnil || !RB_TYPE_P(filename, T_STRING))
   {
     return ST_CONTINUE;
   }

data/lib/datadog/ci/configuration/components.rb

@@ -4,9 +4,13 @@ require "datadog/core/telemetry/ext"
 
 require_relative "../ext/settings"
 require_relative "../git/tree_uploader"
+require_relative "../remote/component"
+require_relative "../remote/library_settings_client"
 require_relative "../test_optimisation/component"
 require_relative "../test_optimisation/coverage/transport"
 require_relative "../test_optimisation/coverage/writer"
+require_relative "../test_retries/component"
+require_relative "../test_retries/null_component"
 require_relative "../test_visibility/component"
 require_relative "../test_visibility/flush"
 require_relative "../test_visibility/null_component"
@@ -15,7 +19,6 @@ require_relative "../test_visibility/serializers/factories/test_suite_level"
 require_relative "../test_visibility/transport"
 require_relative "../transport/adapters/telemetry_webmock_safe_adapter"
 require_relative "../transport/api/builder"
-require_relative "../transport/remote_settings_api"
 require_relative "../utils/identity"
 require_relative "../utils/parsing"
 require_relative "../utils/test_run"
@@ -26,11 +29,14 @@ module Datadog
     module Configuration
       # Adds CI behavior to Datadog trace components
       module Components
-        attr_reader :test_visibility, :test_optimisation
+        attr_reader :test_visibility, :test_optimisation, :git_tree_upload_worker, :ci_remote, :test_retries
 
         def initialize(settings)
           @test_optimisation = nil
           @test_visibility = TestVisibility::NullComponent.new
+          @git_tree_upload_worker = DummyWorker.new
+          @ci_remote = nil
+          @test_retries = TestRetries::NullComponent.new
 
           # Activate CI mode if enabled
           if settings.ci.enabled
@@ -45,6 +51,7 @@ module Datadog
 
           @test_visibility&.shutdown!
           @test_optimisation&.shutdown!
+          @git_tree_upload_worker&.stop
         end
 
         def activate_ci!(settings)
@@ -102,14 +109,19 @@ module Datadog
 
           settings.tracing.test_mode.writer_options = trace_writer_options
 
+          @git_tree_upload_worker = build_git_upload_worker(settings, test_visibility_api)
+          @ci_remote = Remote::Component.new(
+            library_settings_client: build_library_settings_client(settings, test_visibility_api)
+          )
+          @test_retries = TestRetries::Component.new(
+            retry_failed_tests_enabled: settings.ci.retry_failed_tests_enabled,
+            retry_failed_tests_max_attempts: settings.ci.retry_failed_tests_max_attempts,
+            retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit
+          )
           # @type ivar @test_optimisation: Datadog::CI::TestOptimisation::Component
           @test_optimisation = build_test_optimisation(settings, test_visibility_api)
-
           @test_visibility = TestVisibility::Component.new(
-            test_optimisation: @test_optimisation,
-            test_suite_level_visibility_enabled: !settings.ci.force_test_level_visibility,
-            remote_settings_api: build_remote_settings_client(settings, test_visibility_api),
-            git_tree_upload_worker: build_git_upload_worker(settings, test_visibility_api)
+            test_suite_level_visibility_enabled: !settings.ci.force_test_level_visibility
           )
         end
 
@@ -216,8 +228,8 @@ module Datadog
           end
         end
 
-        def build_remote_settings_client(settings, api)
-          Transport::RemoteSettingsApi.new(
+        def build_library_settings_client(settings, api)
+          Remote::LibrarySettingsClient.new(
             api: api,
             dd_env: settings.env,
             config_tags: custom_configuration(settings)
@@ -254,11 +266,13 @@ module Datadog
           # for test visibility we want to enable it by default unless explicitly disabled
           # NOTE: before agentless mode is released, we only enable telemetry when running with Datadog Agent
           env_telemetry_enabled = ENV[Core::Telemetry::Ext::ENV_ENABLED]
-          settings.telemetry.enabled = !settings.ci.agentless_mode_enabled &&
-            (env_telemetry_enabled.nil? || Utils::Parsing.convert_to_bool(env_telemetry_enabled))
+          settings.telemetry.enabled = env_telemetry_enabled.nil? || Utils::Parsing.convert_to_bool(env_telemetry_enabled)
 
           return unless settings.telemetry.enabled
 
+          settings.telemetry.agentless_enabled = true if settings.ci.agentless_mode_enabled
+          settings.telemetry.shutdown_timeout_seconds = 60.0
+
           begin
             require "datadog/core/environment/identity"
             require "datadog/core/telemetry/http/adapters/net"

data/lib/datadog/ci/configuration/settings.rb

@@ -59,7 +59,7 @@ module Datadog
               option :itr_enabled do |o|
                 o.type :bool
                 o.env CI::Ext::Settings::ENV_ITR_ENABLED
-                o.default false
+                o.default true
               end
 
               option :git_metadata_upload_enabled do |o|
@@ -88,6 +88,24 @@ module Datadog
                 o.default true
               end
 
+              option :retry_failed_tests_enabled do |o|
+                o.type :bool
+                o.env CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_ENABLED
+                o.default true
+              end
+
+              option :retry_failed_tests_max_attempts do |o|
+                o.type :int
+                o.env CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS
+                o.default 5
+              end
+
+              option :retry_failed_tests_total_limit do |o|
+                o.type :int
+                o.env CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT
+                o.default 1000
+              end
+
               define_method(:instrument) do |integration_name, options = {}, &block|
                 return unless enabled
 

data/lib/datadog/ci/contrib/cucumber/configuration_override.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "formatter"
+
+module Datadog
+  module CI
+    module Contrib
+      module Cucumber
+        # Changes behaviour of Cucumber::Configuration class
+        module ConfigurationOverride
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          # Instance methods for configuration
+          module InstanceMethods
+            def retry_attempts
+              super if !datadog_test_retries_component&.retry_failed_tests_enabled
+
+              datadog_test_retries_component&.retry_failed_tests_max_attempts
+            end
+
+            def retry_total_tests
+              super if !datadog_test_retries_component&.retry_failed_tests_enabled
+
+              datadog_test_retries_component&.retry_failed_tests_total_limit
+            end
+
+            def datadog_test_retries_component
+              Datadog.send(:components).test_retries
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -20,7 +20,7 @@ module Datadog
 
             @current_test_suite = nil
 
-            @failed_tests_count = 0
+            @failed_test_suites_count = 0
 
             bind_events(config)
           end
@@ -46,10 +46,12 @@ module Datadog
           end
 
           def on_test_run_finished(event)
+            finish_current_test_suite
+
             if event.respond_to?(:success)
               finish_session(event.success)
             else
-              finish_session(@failed_tests_count.zero?)
+              finish_session(@failed_test_suites_count.zero?)
             end
           end
 
@@ -86,7 +88,6 @@ module Datadog
             return if test_span.nil?
 
             finish_span(test_span, event.result)
-            @failed_tests_count += 1 if test_span.failed?
           end
 
           def on_test_step_started(event)
@@ -128,8 +129,6 @@ module Datadog
           end
 
           def finish_session(result)
-            finish_current_test_suite
-
             test_session = test_visibility_component.active_test_session
             test_module = test_visibility_component.active_test_module
 
@@ -155,6 +154,7 @@ module Datadog
 
           def finish_current_test_suite
             @current_test_suite&.finish
+            @failed_test_suites_count += 1 if @current_test_suite&.failed?
 
             @current_test_suite = nil
           end

data/lib/datadog/ci/contrib/cucumber/instrumentation.rb

@@ -6,7 +6,7 @@ module Datadog
   module CI
     module Contrib
       module Cucumber
-        # Instrumentation for Cucumber
+        # Instrumentation for Cucumber::Runtime class
         module Instrumentation
           def self.included(base)
             base.prepend(InstanceMethods)
@@ -21,6 +21,14 @@ module Datadog
               @datadog_formatter ||= CI::Contrib::Cucumber::Formatter.new(@configuration)
               [@datadog_formatter] + existing_formatters
             end
+
+            def begin_scenario(test_case)
+              if Datadog::CI.active_test&.skipped_by_itr?
+                raise ::Cucumber::Core::Test::Result::Skipped, CI::Ext::Test::ITR_TEST_SKIP_REASON
+              end
+
+              super
+            end
           end
         end
       end

data/lib/datadog/ci/contrib/cucumber/patcher.rb

@@ -3,13 +3,13 @@
 require "datadog/tracing/contrib/patcher"
 
 require_relative "instrumentation"
-require_relative "step"
+require_relative "configuration_override"
 
 module Datadog
   module CI
     module Contrib
       module Cucumber
-        # Patcher enables patching of 'cucumber' module.
+        # Patches 'cucumber' gem.
         module Patcher
           include Datadog::Tracing::Contrib::Patcher
 
@@ -21,7 +21,7 @@ module Datadog
 
           def patch
             ::Cucumber::Runtime.include(Instrumentation)
-            ::Cucumber::Core::Test::Step.include(Datadog::CI::Contrib::Cucumber::Step)
+            ::Cucumber::Configuration.include(ConfigurationOverride)
           end
         end
       end

data/lib/datadog/ci/contrib/minitest/runner.rb

@@ -28,6 +28,18 @@ module Datadog
               test_visibility_component.start_test_module(Ext::FRAMEWORK)
             end
 
+            def run_one_method(klass, method_name)
+              return super unless datadog_configuration[:enabled]
+
+              result = nil
+
+              test_retries_component.with_retries do
+                result = super
+              end
+
+              result
+            end
+
             private
 
             def datadog_configuration
@@ -37,6 +49,10 @@ module Datadog
             def test_visibility_component
               Datadog.send(:components).test_visibility
             end
+
+            def test_retries_component
+              Datadog.send(:components).test_retries
+            end
           end
         end
       end

data/lib/datadog/ci/contrib/minitest/test.rb

@@ -71,6 +71,7 @@ module Datadog
               when "S"
                 span.skipped!(reason: failure.message)
               end
+
               span.finish
             end
 

data/lib/datadog/ci/contrib/rspec/example.rb

@@ -34,50 +34,74 @@ module Datadog
 
               if ci_queue?
                 suite_name = "#{suite_name} (ci-queue running example [#{test_name}])"
-                test_suite_span = test_visibility_component.start_test_suite(suite_name)
+                ci_queue_test_span = test_visibility_component.start_test_suite(suite_name)
               end
 
-              test_visibility_component.trace_test(
-                test_name,
-                suite_name,
-                tags: {
-                  CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                  CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
-                  CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(metadata[:file_path]),
-                  CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s,
-                  CI::Ext::Test::TAG_PARAMETERS => Utils::TestRun.test_parameters(
-                    metadata: {"scoped_id" => metadata[:scoped_id]}
-                  )
-                },
-                service: datadog_configuration[:service_name]
-              ) do |test_span|
-                test_span&.itr_unskippable! if metadata[CI::Ext::Test::ITR_UNSKIPPABLE_OPTION]
-
-                metadata[:skip] = CI::Ext::Test::ITR_TEST_SKIP_REASON if test_span&.skipped_by_itr?
-
-                result = super
-
-                case execution_result.status
-                when :passed
-                  test_span&.passed!
-                  test_suite_span&.passed!
-                when :failed
-                  test_span&.failed!(exception: execution_result.exception)
-                  test_suite_span&.failed!
-                else
-                  # :pending or nil
-                  test_span&.skipped!(
-                    reason: execution_result.pending_message,
-                    exception: execution_result.pending_exception
-                  )
-
-                  test_suite_span&.skipped!
+              # don't report test to RSpec::Core::Reporter until retries are done
+              @skip_reporting = true
+
+              test_retries_component.with_retries do
+                test_visibility_component.trace_test(
+                  test_name,
+                  suite_name,
+                  tags: {
+                    CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                    CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::RSpec::Integration.version.to_s,
+                    CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(metadata[:file_path]),
+                    CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s,
+                    CI::Ext::Test::TAG_PARAMETERS => Utils::TestRun.test_parameters(
+                      metadata: {"scoped_id" => metadata[:scoped_id]}
+                    )
+                  },
+                  service: datadog_configuration[:service_name]
+                ) do |test_span|
+                  test_span&.itr_unskippable! if metadata[CI::Ext::Test::ITR_UNSKIPPABLE_OPTION]
+
+                  metadata[:skip] = CI::Ext::Test::ITR_TEST_SKIP_REASON if test_span&.skipped_by_itr?
+
+                  # before each run remove any previous exception
+                  @exception = nil
+
+                  result = super
+
+                  # In case when test job is canceled and RSpec is quitting we don't want to report the last test
+                  # before RSpec context unwinds. This test might have some unrelated errors that we don't want to
+                  # report.
+                  return result if ::RSpec.world.wants_to_quit
+
+                  case execution_result.status
+                  when :passed
+                    test_span&.passed!
+                  when :failed
+                    test_span&.failed!(exception: execution_result.exception)
+                  else
+                    # :pending or nil
+                    test_span&.skipped!(
+                      reason: execution_result.pending_message,
+                      exception: execution_result.pending_exception
+                    )
+                  end
                 end
+              end
 
-                test_suite_span&.finish
+              # after retries are done, we can report the test to RSpec
+              @skip_reporting = false
 
-                result
-              end
+              # this is a special case for ci-queue, we need to finish the test suite span
+              ci_queue_test_span&.finish
+
+              # Finish spec with latest retry's result
+              # TODO: when implementing new test retries make sure to clean @exception before calling this method
+              # if test passed at least once
+              finish(reporter)
+            end
+
+            def finish(reporter)
+              # by default finish test but do not report it to RSpec::Core::Reporter
+              # it is going to be reported once after retries are done
+              return super unless @skip_reporting
+
+              super(::RSpec::Core::NullReporter)
             end
 
             private
@@ -103,6 +127,10 @@ module Datadog
               Datadog.send(:components).test_visibility
             end
 
+            def test_retries_component
+              Datadog.send(:components).test_retries
+            end
+
             def ci_queue?
               !!defined?(::RSpec::Queue::ExampleExtension) &&
                 self.class.ancestors.include?(::RSpec::Queue::ExampleExtension)

data/lib/datadog/ci/contrib/rspec/example_group.rb

@@ -26,7 +26,7 @@ module Datadog
               success = super
               return success unless test_suite
 
-              if success && test_suite.passed_tests_count > 0
+              if success && test_suite.any_passed?
                 test_suite.passed!
               elsif success
                 test_suite.skipped!

data/lib/datadog/ci/ext/settings.rb

@@ -15,6 +15,9 @@ module Datadog
         ENV_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH = "DD_CIVISIBILITY_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH"
         ENV_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE = "DD_CIVISIBILITY_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE"
         ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING = "DD_CIVISIBILITY_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING"
+        ENV_RETRY_FAILED_TESTS_ENABLED = "DD_CIVISIBILITY_FLAKY_RETRY_ENABLED"
+        ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS = "DD_CIVISIBILITY_FLAKY_RETRY_COUNT"
+        ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT = "DD_CIVISIBILITY_TOTAL_FLAKY_RETRY_COUNT"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = %w[

data/lib/datadog/ci/ext/telemetry.rb

@@ -62,6 +62,7 @@ module Datadog
         TAG_IS_UNSUPPORTED_CI = "is_unsupported_ci"
         TAG_BROWSER_DRIVER = "browser_driver"
         TAG_IS_RUM = "is_rum"
+        TAG_IS_RETRY = "is_retry"
         TAG_LIBRARY = "library"
         TAG_ENDPOINT = "endpoint"
         TAG_ERROR_TYPE = "error_type"

data/lib/datadog/ci/ext/test.rb

@@ -8,7 +8,7 @@ module Datadog
       module Test
         CONTEXT_ORIGIN = "ciapp-test"
 
-        # Base test visibility tags
+        # Test visibility tags
         TAG_FRAMEWORK = "test.framework"
         TAG_FRAMEWORK_VERSION = "test.framework_version"
         TAG_NAME = "test.name"
@@ -23,7 +23,8 @@ module Datadog
         TAG_CODEOWNERS = "test.codeowners"
         TAG_PARAMETERS = "test.parameters"
 
-        # ITR tags
+        # Test optimisation tags
+        TAG_CODE_COVERAGE_ENABLED = "test.code_coverage.enabled"
         TAG_ITR_TEST_SKIPPING_ENABLED = "test.itr.tests_skipping.enabled"
         TAG_ITR_TEST_SKIPPING_TYPE = "test.itr.tests_skipping.type"
         TAG_ITR_TEST_SKIPPING_COUNT = "test.itr.tests_skipping.count"
@@ -32,11 +33,9 @@ module Datadog
         TAG_ITR_UNSKIPPABLE = "test.itr.unskippable"
         TAG_ITR_FORCED_RUN = "test.itr.forced_run"
 
-        # Code coverage tags
-        TAG_CODE_COVERAGE_ENABLED = "test.code_coverage.enabled"
-
-        # Special tags, not sent to the backend.
-        # these tags are special and used to correlate tests with the test sessions, suites, and modules
+        # Internal tags, they are not sent to the backend.
+        # These tags are internal to this library and used to correlate tests with
+        # the test sessions, suites, and modules.
         TAG_TEST_SESSION_ID = "_test.session_id"
         TAG_TEST_MODULE_ID = "_test.module_id"
         TAG_TEST_SUITE_ID = "_test.suite_id"
@@ -50,8 +49,7 @@ module Datadog
         TAG_RUNTIME_VERSION = "runtime.version"
 
         # Tags for browser tests
-        # true if Datadog RUM was detected in the page(s) loaded by Selenium
-        TAG_IS_RUM_ACTIVE = "test.is_rum_active"
+        TAG_IS_RUM_ACTIVE = "test.is_rum_active" # true if Datadog RUM was detected in the page(s) loaded by Selenium
         TAG_BROWSER_DRIVER = "test.browser.driver"
         # version of selenium driver used
         TAG_BROWSER_DRIVER_VERSION = "test.browser.driver_version"
@@ -60,6 +58,9 @@ module Datadog
         # version of the browser, if multiple browsers or multiple versions then this tag is empty
         TAG_BROWSER_VERSION = "test.browser.version"
 
+        # Tags for test retries
+        TAG_IS_RETRY = "test.is_retry" # true if test was retried by datadog-ci library
+
         # internal APM tag to mark a span as a test span
         TAG_SPAN_KIND = "span.kind"
         SPAN_KIND_TEST = "test"
@@ -68,8 +69,7 @@ module Datadog
         INHERITABLE_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION].freeze
 
         # could be either "test" or "suite" depending on whether we skip individual tests or whole suites
-        # we use test skipping for Ruby
-        ITR_TEST_SKIPPING_MODE = "test"
+        ITR_TEST_SKIPPING_MODE = "test" # we always skip tests (not suites) in Ruby
         ITR_TEST_SKIP_REASON = "Skipped by Datadog's intelligent test runner"
         ITR_UNSKIPPABLE_OPTION = :datadog_itr_unskippable
 

data/lib/datadog/ci/ext/transport.rb

@@ -36,6 +36,7 @@ module Datadog
         DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY = "code_coverage"
         DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY = "tests_skipping"
         DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY = "require_git"
+        DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY = "flaky_test_retries_enabled"
         DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze
 
         DD_API_GIT_SEARCH_COMMITS_PATH = "/api/v2/git/repository/search_commits"