datadog-ci 1.4.1 → 1.6.0

data/lib/datadog/ci/test_retries/unique_tests_client.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "../ext/telemetry"
+require_relative "../ext/transport"
+require_relative "../transport/telemetry"
+require_relative "../utils/telemetry"
+require_relative "../utils/test_run"
+
+module Datadog
+  module CI
+    module TestRetries
+      # fetch a list of unique known tests from the backend
+      class UniqueTestsClient
+        class Response
+          def initialize(http_response)
+            @http_response = http_response
+            @json = nil
+          end
+
+          def ok?
+            resp = @http_response
+            !resp.nil? && resp.ok?
+          end
+
+          def tests
+            res = Set.new
+
+            payload
+              .fetch("data", {})
+              .fetch("attributes", {})
+              .fetch("tests", {})
+              .each do |_test_module, suites_hash|
+                suites_hash.each do |test_suite, tests|
+                  tests.each do |test_name|
+                    res << Utils::TestRun.datadog_test_id(test_name, test_suite)
+                  end
+                end
+              end
+
+            res
+          end
+
+          private
+
+          def payload
+            cached = @json
+            return cached unless cached.nil?
+
+            resp = @http_response
+            return @json = {} if resp.nil? || !ok?
+
+            begin
+              @json = JSON.parse(resp.payload)
+            rescue JSON::ParserError => e
+              Datadog.logger.error("Failed to parse unique known tests response payload: #{e}. Payload was: #{resp.payload}")
+              @json = {}
+            end
+          end
+        end
+
+        def initialize(dd_env:, api: nil, config_tags: {})
+          @api = api
+          @dd_env = dd_env
+          @config_tags = config_tags
+        end
+
+        def fetch_unique_tests(test_session)
+          api = @api
+          return Set.new unless api
+
+          request_payload = payload(test_session)
+          Datadog.logger.debug("Fetching unique known tests with request: #{request_payload}")
+
+          http_response = api.api_request(
+            path: Ext::Transport::DD_API_UNIQUE_TESTS_PATH,
+            payload: request_payload
+          )
+
+          Transport::Telemetry.api_requests(
+            Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST,
+            1,
+            compressed: http_response.request_compressed
+          )
+          Utils::Telemetry.distribution(Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_MS, http_response.duration_ms)
+          Utils::Telemetry.distribution(
+            Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_BYTES,
+            http_response.response_size.to_f,
+            {Ext::Telemetry::TAG_RESPONSE_COMPRESSED => http_response.gzipped_content?.to_s}
+          )
+
+          unless http_response.ok?
+            Transport::Telemetry.api_requests_errors(
+              Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_ERRORS,
+              1,
+              error_type: http_response.telemetry_error_type,
+              status_code: http_response.code
+            )
+          end
+
+          Response.new(http_response).tests
+        end
+
+        private
+
+        def payload(test_session)
+          {
+            "data" => {
+              "id" => Datadog::Core::Environment::Identity.id,
+              "type" => Ext::Transport::DD_API_UNIQUE_TESTS_TYPE,
+              "attributes" => {
+                "repository_url" => test_session.git_repository_url,
+                "service" => test_session.service,
+                "env" => @dd_env,
+                "sha" => test_session.git_commit_sha,
+                "configurations" => {
+                  Ext::Test::TAG_OS_PLATFORM => test_session.os_platform,
+                  Ext::Test::TAG_OS_ARCHITECTURE => test_session.os_architecture,
+                  Ext::Test::TAG_OS_VERSION => test_session.os_version,
+                  Ext::Test::TAG_RUNTIME_NAME => test_session.runtime_name,
+                  Ext::Test::TAG_RUNTIME_VERSION => test_session.runtime_version,
+                  "custom" => @config_tags
+                }
+              }
+            }
+          }.to_json
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_session.rb

@@ -12,6 +12,8 @@ module Datadog
     #
     # @public_api
     class TestSession < ConcurrentSpan
+      attr_accessor :total_tests_count
+
       # Finishes the current test session.
       # @return [void]
       def finish
@@ -26,6 +28,24 @@ module Datadog
         get_tag(Ext::Test::TAG_COMMAND)
       end
 
+      # Return the test session's command used to run the tests
+      # @return [String] the command for this test session.
+      def test_command
+        get_tag(Ext::Test::TAG_COMMAND)
+      end
+
+      # Return the test session's CI provider name (e.g. "travis", "circleci", etc.)
+      # @return [String] the provider name for this test session.
+      def ci_provider
+        get_tag(Ext::Environment::TAG_PROVIDER_NAME)
+      end
+
+      # Return the test session's CI job name (e.g. "build", "test", etc.)
+      # @return [String] the job name for this test session.
+      def ci_job_name
+        get_tag(Ext::Environment::TAG_JOB_NAME)
+      end
+
       def skipping_tests?
         get_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_ENABLED) == "true"
       end

data/lib/datadog/ci/test_suite.rb

@@ -51,6 +51,14 @@ module Datadog
         end
       end
 
+      # @internal
+      def any_test_retry_passed?(test_id)
+        synchronize do
+          stats = @execution_stats_per_test[test_id]
+          stats && stats[Ext::Test::Status::PASS] > 0
+        end
+      end
+
       # @internal
       def test_executed?(test_id)
         synchronize do

data/lib/datadog/ci/test_visibility/component.rb

@@ -17,23 +17,25 @@ module Datadog
     module TestVisibility
       # Common behavior for CI tests
       class Component
-        attr_reader :test_suite_level_visibility_enabled
-
-        FIBER_LOCAL_TEST_FINISHED_CALLBACK_KEY = :__dd_test_finished_callback
+        attr_reader :test_suite_level_visibility_enabled, :logical_test_session_name
 
         def initialize(
           test_suite_level_visibility_enabled: false,
-          codeowners: Codeowners::Parser.new(Git::LocalRepository.root).parse
+          codeowners: Codeowners::Parser.new(Git::LocalRepository.root).parse,
+          logical_test_session_name: nil
         )
           @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled
           @context = Context.new
           @codeowners = codeowners
+          @logical_test_session_name = logical_test_session_name
         end
 
-        def start_test_session(service: nil, tags: {})
+        def start_test_session(service: nil, tags: {}, total_tests_count: 0)
           return skip_tracing unless test_suite_level_visibility_enabled
 
           test_session = @context.start_test_session(service: service, tags: tags)
+          test_session.total_tests_count = total_tests_count
+
           on_test_session_started(test_session)
           test_session
         end
@@ -57,6 +59,8 @@ module Datadog
         def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
           if block
             @context.trace_test(test_name, test_suite_name, service: service, tags: tags) do |test|
+              subscribe_to_after_stop_event(test.tracer_span)
+
               on_test_started(test)
               res = block.call(test)
               on_test_finished(test)
@@ -64,6 +68,7 @@ module Datadog
             end
           else
             test = @context.trace_test(test_name, test_suite_name, service: service, tags: tags)
+            subscribe_to_after_stop_event(test.tracer_span)
             on_test_started(test)
             test
           end
@@ -127,15 +132,6 @@ module Datadog
           @context.deactivate_test_suite(test_suite_name)
         end
 
-        # sets fiber-local callback to be called when test is finished
-        def set_test_finished_callback(callback)
-          Thread.current[FIBER_LOCAL_TEST_FINISHED_CALLBACK_KEY] = callback
-        end
-
-        def remove_test_finished_callback
-          Thread.current[FIBER_LOCAL_TEST_FINISHED_CALLBACK_KEY] = nil
-        end
-
         def itr_enabled?
           test_optimisation.enabled?
         end
@@ -158,6 +154,9 @@ module Datadog
           Telemetry.test_session_started(test_session)
           Telemetry.event_created(test_session)
 
+          # sets logical test session name if none provided by the user
+          override_logical_test_session_name!(test_session) if logical_test_session_name.nil?
+
           # signal Remote::Component to configure the library
           remote.configure(test_session)
         end
@@ -204,7 +203,11 @@ module Datadog
 
           Telemetry.event_finished(test)
 
-          Thread.current[FIBER_LOCAL_TEST_FINISHED_CALLBACK_KEY]&.call(test)
+          test_retries.record_test_finished(test)
+        end
+
+        def on_after_test_span_finished(tracer_span)
+          test_retries.record_test_span_duration(tracer_span)
         end
 
         # HELPERS
@@ -212,6 +215,14 @@ module Datadog
           block&.call(nil)
         end
 
+        def subscribe_to_after_stop_event(tracer_span)
+          events = tracer_span.send(:events)
+
+          events.after_stop.subscribe do |span|
+            on_after_test_span_finished(span)
+          end
+        end
+
         def set_codeowners(test)
           source = test.source_file
           owners = @codeowners.list_owners(source) if source
@@ -263,10 +274,22 @@ module Datadog
           end
         end
 
+        def override_logical_test_session_name!(test_session)
+          @logical_test_session_name = test_session.test_command
+          ci_job_name = test_session.ci_job_name
+          if ci_job_name
+            @logical_test_session_name = "#{ci_job_name}-#{@logical_test_session_name}"
+          end
+        end
+
         def test_optimisation
           Datadog.send(:components).test_optimisation
         end
 
+        def test_retries
+          Datadog.send(:components).test_retries
+        end
+
         def git_tree_upload_worker
           Datadog.send(:components).git_tree_upload_worker
         end

data/lib/datadog/ci/test_visibility/context.rb

@@ -11,6 +11,8 @@ require_relative "../ext/app_types"
 require_relative "../ext/environment"
 require_relative "../ext/test"
 
+require_relative "../utils/test_run"
+
 require_relative "../span"
 require_relative "../test"
 require_relative "../test_session"
@@ -203,6 +205,8 @@ module Datadog
 
           ci_span.set_tags(tags)
           ci_span.set_tags(@environment_tags)
+
+          ci_span.set_metric(Ext::Test::METRIC_CPU_COUNT, Utils::TestRun.virtual_cpu_count)
         end
 
         # PROPAGATING CONTEXT FROM TOP-LEVEL TO THE LOWER LEVELS

data/lib/datadog/ci/test_visibility/null_component.rb

@@ -5,7 +5,7 @@ module Datadog
     module TestVisibility
       # Special test visibility component that does not record anything
       class NullComponent
-        def start_test_session(service: nil, tags: {})
+        def start_test_session(service: nil, tags: {}, total_tests_count: 0)
           skip_tracing
         end
 
@@ -53,6 +53,13 @@ module Datadog
         def remove_test_finished_callback
         end
 
+        def test_suite_level_visibility_enabled
+          false
+        end
+
+        def logical_test_session_name
+        end
+
         private
 
         def skip_tracing(block = nil)

data/lib/datadog/ci/test_visibility/telemetry.rb

@@ -34,9 +34,7 @@ module Datadog
             1,
             {
               Ext::Telemetry::TAG_AUTO_INJECTED => "false", # ruby doesn't support auto injection yet
-              Ext::Telemetry::TAG_PROVIDER =>
-                test_session.get_tag(Ext::Environment::TAG_PROVIDER_NAME) ||
-                Ext::Telemetry::Provider::UNSUPPORTED
+              Ext::Telemetry::TAG_PROVIDER => test_session.ci_provider || Ext::Telemetry::Provider::UNSUPPORTED
             }
           )
         end
@@ -58,6 +56,15 @@ module Datadog
           # set is_retry tag if span represents a retried test
           tags[Ext::Telemetry::TAG_IS_RETRY] = "true" if span.get_tag(Ext::Test::TAG_IS_RETRY)
 
+          # is_new
+          tags[Ext::Telemetry::TAG_IS_NEW] = "true" if span.get_tag(Ext::Test::TAG_IS_NEW)
+
+          # session-level tag - early_flake_detection_abort_reason
+          early_flake_detection_abort_reason = span.get_tag(Ext::Test::TAG_EARLY_FLAKE_ABORT_REASON)
+          if early_flake_detection_abort_reason
+            tags[Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ABORT_REASON] = early_flake_detection_abort_reason
+          end
+
           tags
         end
 

data/lib/datadog/ci/test_visibility/transport.rb

@@ -3,6 +3,7 @@
 require "datadog/core/environment/identity"
 
 require_relative "serializers/factories/test_level"
+require_relative "../ext/app_types"
 require_relative "../ext/telemetry"
 require_relative "../ext/transport"
 require_relative "../transport/event_platform_transport"
@@ -51,7 +52,12 @@ module Datadog
         end
 
         def encode_span(trace, span)
-          serializer = serializers_factory.serializer(trace, span, options: {itr_correlation_id: itr&.correlation_id})
+          serializer = serializers_factory.serializer(
+            trace,
+            span,
+            options: {itr_correlation_id: test_optimisation&.correlation_id}
+          )
+
           if serializer.valid?
             encoded = encoder.encode(serializer)
             return nil if event_too_large?(span, encoded)
@@ -75,7 +81,7 @@ module Datadog
           packer.write(1)
 
           packer.write("metadata")
-          packer.write_map_header(1)
+          packer.write_map_header(1 + Ext::AppTypes::CI_SPAN_TYPES.size)
 
           packer.write("*")
           metadata_fields_count = dd_env ? 4 : 3
@@ -95,12 +101,24 @@ module Datadog
           packer.write("library_version")
           packer.write(Datadog::CI::VERSION::STRING)
 
+          Ext::AppTypes::CI_SPAN_TYPES.each do |ci_span_type|
+            packer.write(ci_span_type)
+            packer.write_map_header(1)
+
+            packer.write(Ext::Test::METADATA_TAG_TEST_SESSION_NAME)
+            packer.write(test_visibility&.logical_test_session_name)
+          end
+
           packer.write("events")
         end
 
-        def itr
+        def test_optimisation
           @test_optimisation ||= Datadog::CI.send(:test_optimisation)
         end
+
+        def test_visibility
+          @test_visibility ||= Datadog::CI.send(:test_visibility)
+        end
       end
     end
   end

data/lib/datadog/ci/utils/test_run.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "etc"
+
 module Datadog
   module CI
     module Utils
@@ -10,7 +12,7 @@ module Datadog
           @command = "#{$0} #{ARGV.join(" ")}"
         end
 
-        def self.skippable_test_id(test_name, suite, parameters = nil)
+        def self.datadog_test_id(test_name, suite, parameters = nil)
           "#{suite}.#{test_name}.#{parameters}"
         end
 
@@ -34,6 +36,12 @@ module Datadog
           end
           res
         end
+
+        def self.virtual_cpu_count
+          return @virtual_cpu_count if defined?(@virtual_cpu_count)
+
+          @virtual_cpu_count = ::Etc.nprocessors
+        end
       end
     end
   end

data/lib/datadog/ci/version.rb

@@ -4,8 +4,8 @@ module Datadog
   module CI
     module VERSION
       MAJOR = 1
-      MINOR = 4
-      PATCH = 1
+      MINOR = 6
+      PATCH = 0
       PRE = nil
       BUILD = nil
       # PRE and BUILD above are modified for dev gems during gem build GHA workflow

data/lib/datadog/ci.rb

@@ -6,6 +6,7 @@ require_relative "ci/utils/telemetry"
 require_relative "ci/ext/app_types"
 require_relative "ci/ext/telemetry"
 
+require "datadog"
 require "datadog/core"
 
 module Datadog
@@ -27,7 +28,8 @@ module Datadog
       # ```
       # Datadog::CI.start_test_session(
       #   service: "my-web-site-tests",
-      #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" }
+      #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" },
+      #   total_tests_count: 100
       # )
       #
       # # Somewhere else after test run has ended
@@ -38,15 +40,16 @@ module Datadog
       #
       # @param [String] service the service name for this session (optional, defaults to DD_SERVICE or repository name)
       # @param [Hash<String,String>] tags extra tags which should be added to the test session.
+      # @param [Integer] total_tests_count the total number of tests in the test session (optional, defaults to 0) - it is used to limit the number of new tests retried within session if early flake detection is enabled
       # @return [Datadog::CI::TestSession] the active, running {Datadog::CI::TestSession}.
       # @return [nil] if test suite level visibility is disabled or CI mode is disabled.
-      def start_test_session(service: Utils::Configuration.fetch_service_name("test"), tags: {})
+      def start_test_session(service: Utils::Configuration.fetch_service_name("test"), tags: {}, total_tests_count: 0)
         Utils::Telemetry.inc(
           Ext::Telemetry::METRIC_MANUAL_API_EVENTS,
           1,
           {Ext::Telemetry::TAG_EVENT_TYPE => Ext::Telemetry::EventType::SESSION}
         )
-        test_visibility.start_test_session(service: service, tags: tags)
+        test_visibility.start_test_session(service: service, tags: tags, total_tests_count: total_tests_count)
       end
 
       # The active, unfinished test session.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datadog-ci
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.6.0
 platform: ruby
 authors:
 - Datadog, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-08-28 00:00:00.000000000 Z
+date: 2024-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: datadog
@@ -68,8 +68,8 @@ files:
 - lib/datadog/ci/configuration/settings.rb
 - lib/datadog/ci/contrib/contrib.rb
 - lib/datadog/ci/contrib/cucumber/configuration/settings.rb
-- lib/datadog/ci/contrib/cucumber/configuration_override.rb
 - lib/datadog/ci/contrib/cucumber/ext.rb
+- lib/datadog/ci/contrib/cucumber/filter.rb
 - lib/datadog/ci/contrib/cucumber/formatter.rb
 - lib/datadog/ci/contrib/cucumber/instrumentation.rb
 - lib/datadog/ci/contrib/cucumber/integration.rb
@@ -139,6 +139,7 @@ files:
 - lib/datadog/ci/remote/component.rb
 - lib/datadog/ci/remote/library_settings.rb
 - lib/datadog/ci/remote/library_settings_client.rb
+- lib/datadog/ci/remote/slow_test_retries.rb
 - lib/datadog/ci/span.rb
 - lib/datadog/ci/test.rb
 - lib/datadog/ci/test_module.rb
@@ -150,10 +151,16 @@ files:
 - lib/datadog/ci/test_optimisation/skippable.rb
 - lib/datadog/ci/test_optimisation/telemetry.rb
 - lib/datadog/ci/test_retries/component.rb
+- lib/datadog/ci/test_retries/driver/base.rb
+- lib/datadog/ci/test_retries/driver/no_retry.rb
+- lib/datadog/ci/test_retries/driver/retry_failed.rb
+- lib/datadog/ci/test_retries/driver/retry_new.rb
 - lib/datadog/ci/test_retries/null_component.rb
 - lib/datadog/ci/test_retries/strategy/base.rb
 - lib/datadog/ci/test_retries/strategy/no_retry.rb
 - lib/datadog/ci/test_retries/strategy/retry_failed.rb
+- lib/datadog/ci/test_retries/strategy/retry_new.rb
+- lib/datadog/ci/test_retries/unique_tests_client.rb
 - lib/datadog/ci/test_session.rb
 - lib/datadog/ci/test_suite.rb
 - lib/datadog/ci/test_visibility/component.rb
@@ -187,7 +194,6 @@ files:
 - lib/datadog/ci/utils/bundle.rb
 - lib/datadog/ci/utils/configuration.rb
 - lib/datadog/ci/utils/git.rb
-- lib/datadog/ci/utils/identity.rb
 - lib/datadog/ci/utils/parsing.rb
 - lib/datadog/ci/utils/telemetry.rb
 - lib/datadog/ci/utils/test_run.rb
@@ -219,7 +225,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 2.0.0
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.16
 signing_key: 
 specification_version: 4
 summary: Datadog CI visibility for your ruby application

data/lib/datadog/ci/contrib/cucumber/configuration_override.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "formatter"
-
-module Datadog
-  module CI
-    module Contrib
-      module Cucumber
-        # Changes behaviour of Cucumber::Configuration class
-        module ConfigurationOverride
-          def self.included(base)
-            base.prepend(InstanceMethods)
-          end
-
-          # Instance methods for configuration
-          module InstanceMethods
-            def retry_attempts
-              super if !datadog_test_retries_component&.retry_failed_tests_enabled
-
-              datadog_test_retries_component&.retry_failed_tests_max_attempts
-            end
-
-            def retry_total_tests
-              super if !datadog_test_retries_component&.retry_failed_tests_enabled
-
-              datadog_test_retries_component&.retry_failed_tests_total_limit
-            end
-
-            def datadog_test_retries_component
-              Datadog.send(:components).test_retries
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/datadog/ci/utils/identity.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module Datadog
-  module CI
-    module Utils
-      module Identity
-        def self.included(base)
-          base.singleton_class.prepend(ClassMethods)
-        end
-
-        module ClassMethods
-          # return datadog-ci gem version instead of datadog gem version
-          def gem_datadog_version
-            Datadog::CI::VERSION::STRING
-          end
-        end
-      end
-    end
-  end
-end