datadog-ci 1.4.0 → 1.5.0

data/lib/datadog/ci/test_retries/unique_tests_client.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "../ext/telemetry"
+require_relative "../ext/transport"
+require_relative "../transport/telemetry"
+require_relative "../utils/telemetry"
+require_relative "../utils/test_run"
+
+module Datadog
+  module CI
+    module TestRetries
+      # fetch a list of unique known tests from the backend
+      class UniqueTestsClient
+        class Response
+          def initialize(http_response)
+            @http_response = http_response
+            @json = nil
+          end
+
+          def ok?
+            resp = @http_response
+            !resp.nil? && resp.ok?
+          end
+
+          def tests
+            res = Set.new
+
+            payload
+              .fetch("data", {})
+              .fetch("attributes", {})
+              .fetch("tests", {})
+              .each do |_test_module, suites_hash|
+                suites_hash.each do |test_suite, tests|
+                  tests.each do |test_name|
+                    res << Utils::TestRun.datadog_test_id(test_name, test_suite)
+                  end
+                end
+              end
+
+            res
+          end
+
+          private
+
+          def payload
+            cached = @json
+            return cached unless cached.nil?
+
+            resp = @http_response
+            return @json = {} if resp.nil? || !ok?
+
+            begin
+              @json = JSON.parse(resp.payload)
+            rescue JSON::ParserError => e
+              Datadog.logger.error("Failed to parse unique known tests response payload: #{e}. Payload was: #{resp.payload}")
+              @json = {}
+            end
+          end
+        end
+
+        def initialize(dd_env:, api: nil, config_tags: {})
+          @api = api
+          @dd_env = dd_env
+          @config_tags = config_tags
+        end
+
+        def fetch_unique_tests(test_session)
+          api = @api
+          return Set.new unless api
+
+          request_payload = payload(test_session)
+          Datadog.logger.debug("Fetching unique known tests with request: #{request_payload}")
+
+          http_response = api.api_request(
+            path: Ext::Transport::DD_API_UNIQUE_TESTS_PATH,
+            payload: request_payload
+          )
+
+          Transport::Telemetry.api_requests(
+            Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST,
+            1,
+            compressed: http_response.request_compressed
+          )
+          Utils::Telemetry.distribution(Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_MS, http_response.duration_ms)
+          Utils::Telemetry.distribution(
+            Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_RESPONSE_BYTES,
+            http_response.response_size.to_f,
+            {Ext::Telemetry::TAG_RESPONSE_COMPRESSED => http_response.gzipped_content?.to_s}
+          )
+
+          unless http_response.ok?
+            Transport::Telemetry.api_requests_errors(
+              Ext::Telemetry::METRIC_EFD_UNIQUE_TESTS_REQUEST_ERRORS,
+              1,
+              error_type: http_response.telemetry_error_type,
+              status_code: http_response.code
+            )
+          end
+
+          Response.new(http_response).tests
+        end
+
+        private
+
+        def payload(test_session)
+          {
+            "data" => {
+              "id" => Datadog::Core::Environment::Identity.id,
+              "type" => Ext::Transport::DD_API_UNIQUE_TESTS_TYPE,
+              "attributes" => {
+                "repository_url" => test_session.git_repository_url,
+                "service" => test_session.service,
+                "env" => @dd_env,
+                "sha" => test_session.git_commit_sha,
+                "configurations" => {
+                  Ext::Test::TAG_OS_PLATFORM => test_session.os_platform,
+                  Ext::Test::TAG_OS_ARCHITECTURE => test_session.os_architecture,
+                  Ext::Test::TAG_OS_VERSION => test_session.os_version,
+                  Ext::Test::TAG_RUNTIME_NAME => test_session.runtime_name,
+                  Ext::Test::TAG_RUNTIME_VERSION => test_session.runtime_version,
+                  "custom" => @config_tags
+                }
+              }
+            }
+          }.to_json
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_session.rb

@@ -12,6 +12,8 @@ module Datadog
     #
     # @public_api
     class TestSession < ConcurrentSpan
+      attr_accessor :total_tests_count
+
       # Finishes the current test session.
       # @return [void]
       def finish

data/lib/datadog/ci/test_suite.rb

@@ -51,6 +51,14 @@ module Datadog
         end
       end
 
+      # @internal
+      def any_test_retry_passed?(test_id)
+        synchronize do
+          stats = @execution_stats_per_test[test_id]
+          stats && stats[Ext::Test::Status::PASS] > 0
+        end
+      end
+
       # @internal
       def test_executed?(test_id)
         synchronize do

data/lib/datadog/ci/test_visibility/component.rb

@@ -19,8 +19,6 @@ module Datadog
       class Component
         attr_reader :test_suite_level_visibility_enabled
 
-        FIBER_LOCAL_TEST_FINISHED_CALLBACK_KEY = :__dd_test_finished_callback
-
         def initialize(
           test_suite_level_visibility_enabled: false,
           codeowners: Codeowners::Parser.new(Git::LocalRepository.root).parse
@@ -30,10 +28,12 @@ module Datadog
           @codeowners = codeowners
         end
 
-        def start_test_session(service: nil, tags: {})
+        def start_test_session(service: nil, tags: {}, total_tests_count: 0)
           return skip_tracing unless test_suite_level_visibility_enabled
 
           test_session = @context.start_test_session(service: service, tags: tags)
+          test_session.total_tests_count = total_tests_count
+
           on_test_session_started(test_session)
           test_session
         end
@@ -57,6 +57,8 @@ module Datadog
         def trace_test(test_name, test_suite_name, service: nil, tags: {}, &block)
           if block
             @context.trace_test(test_name, test_suite_name, service: service, tags: tags) do |test|
+              subscribe_to_after_stop_event(test.tracer_span)
+
               on_test_started(test)
               res = block.call(test)
               on_test_finished(test)
@@ -64,6 +66,7 @@ module Datadog
             end
           else
             test = @context.trace_test(test_name, test_suite_name, service: service, tags: tags)
+            subscribe_to_after_stop_event(test.tracer_span)
             on_test_started(test)
             test
           end
@@ -127,15 +130,6 @@ module Datadog
           @context.deactivate_test_suite(test_suite_name)
         end
 
-        # sets fiber-local callback to be called when test is finished
-        def set_test_finished_callback(callback)
-          Thread.current[FIBER_LOCAL_TEST_FINISHED_CALLBACK_KEY] = callback
-        end
-
-        def remove_test_finished_callback
-          Thread.current[FIBER_LOCAL_TEST_FINISHED_CALLBACK_KEY] = nil
-        end
-
         def itr_enabled?
           test_optimisation.enabled?
         end
@@ -204,7 +198,11 @@ module Datadog
 
           Telemetry.event_finished(test)
 
-          Thread.current[FIBER_LOCAL_TEST_FINISHED_CALLBACK_KEY]&.call(test)
+          test_retries.record_test_finished(test)
+        end
+
+        def on_after_test_span_finished(tracer_span)
+          test_retries.record_test_span_duration(tracer_span)
         end
 
         # HELPERS
@@ -212,6 +210,14 @@ module Datadog
           block&.call(nil)
         end
 
+        def subscribe_to_after_stop_event(tracer_span)
+          events = tracer_span.send(:events)
+
+          events.after_stop.subscribe do |span|
+            on_after_test_span_finished(span)
+          end
+        end
+
         def set_codeowners(test)
           source = test.source_file
           owners = @codeowners.list_owners(source) if source
@@ -267,6 +273,10 @@ module Datadog
           Datadog.send(:components).test_optimisation
         end
 
+        def test_retries
+          Datadog.send(:components).test_retries
+        end
+
         def git_tree_upload_worker
           Datadog.send(:components).git_tree_upload_worker
         end

data/lib/datadog/ci/test_visibility/null_component.rb

@@ -5,7 +5,7 @@ module Datadog
     module TestVisibility
       # Special test visibility component that does not record anything
       class NullComponent
-        def start_test_session(service: nil, tags: {})
+        def start_test_session(service: nil, tags: {}, total_tests_count: 0)
           skip_tracing
         end
 

data/lib/datadog/ci/test_visibility/telemetry.rb

@@ -58,6 +58,15 @@ module Datadog
           # set is_retry tag if span represents a retried test
           tags[Ext::Telemetry::TAG_IS_RETRY] = "true" if span.get_tag(Ext::Test::TAG_IS_RETRY)
 
+          # is_new
+          tags[Ext::Telemetry::TAG_IS_NEW] = "true" if span.get_tag(Ext::Test::TAG_IS_NEW)
+
+          # session-level tag - early_flake_detection_abort_reason
+          early_flake_detection_abort_reason = span.get_tag(Ext::Test::TAG_EARLY_FLAKE_ABORT_REASON)
+          if early_flake_detection_abort_reason
+            tags[Ext::Telemetry::TAG_EARLY_FLAKE_DETECTION_ABORT_REASON] = early_flake_detection_abort_reason
+          end
+
           tags
         end
 

data/lib/datadog/ci/utils/test_run.rb

@@ -10,7 +10,7 @@ module Datadog
           @command = "#{$0} #{ARGV.join(" ")}"
         end
 
-        def self.skippable_test_id(test_name, suite, parameters = nil)
+        def self.datadog_test_id(test_name, suite, parameters = nil)
           "#{suite}.#{test_name}.#{parameters}"
         end
 

data/lib/datadog/ci/version.rb

@@ -4,7 +4,7 @@ module Datadog
   module CI
     module VERSION
       MAJOR = 1
-      MINOR = 4
+      MINOR = 5
       PATCH = 0
       PRE = nil
       BUILD = nil

data/lib/datadog/ci.rb

@@ -6,6 +6,7 @@ require_relative "ci/utils/telemetry"
 require_relative "ci/ext/app_types"
 require_relative "ci/ext/telemetry"
 
+require "datadog"
 require "datadog/core"
 
 module Datadog
@@ -27,7 +28,8 @@ module Datadog
       # ```
       # Datadog::CI.start_test_session(
       #   service: "my-web-site-tests",
-      #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" }
+      #   tags: { Datadog::CI::Ext::Test::TAG_FRAMEWORK => "my-test-framework" },
+      #   total_tests_count: 100
       # )
       #
       # # Somewhere else after test run has ended
@@ -38,15 +40,16 @@ module Datadog
       #
       # @param [String] service the service name for this session (optional, defaults to DD_SERVICE or repository name)
       # @param [Hash<String,String>] tags extra tags which should be added to the test session.
+      # @param [Integer] total_tests_count the total number of tests in the test session (optional, defaults to 0) - it is used to limit the number of new tests retried within session if early flake detection is enabled
       # @return [Datadog::CI::TestSession] the active, running {Datadog::CI::TestSession}.
       # @return [nil] if test suite level visibility is disabled or CI mode is disabled.
-      def start_test_session(service: Utils::Configuration.fetch_service_name("test"), tags: {})
+      def start_test_session(service: Utils::Configuration.fetch_service_name("test"), tags: {}, total_tests_count: 0)
         Utils::Telemetry.inc(
           Ext::Telemetry::METRIC_MANUAL_API_EVENTS,
           1,
           {Ext::Telemetry::TAG_EVENT_TYPE => Ext::Telemetry::EventType::SESSION}
         )
-        test_visibility.start_test_session(service: service, tags: tags)
+        test_visibility.start_test_session(service: service, tags: tags, total_tests_count: total_tests_count)
       end
 
       # The active, unfinished test session.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datadog-ci
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Datadog, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-08-26 00:00:00.000000000 Z
+date: 2024-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: datadog
@@ -68,8 +68,8 @@ files:
 - lib/datadog/ci/configuration/settings.rb
 - lib/datadog/ci/contrib/contrib.rb
 - lib/datadog/ci/contrib/cucumber/configuration/settings.rb
-- lib/datadog/ci/contrib/cucumber/configuration_override.rb
 - lib/datadog/ci/contrib/cucumber/ext.rb
+- lib/datadog/ci/contrib/cucumber/filter.rb
 - lib/datadog/ci/contrib/cucumber/formatter.rb
 - lib/datadog/ci/contrib/cucumber/instrumentation.rb
 - lib/datadog/ci/contrib/cucumber/integration.rb
@@ -139,6 +139,7 @@ files:
 - lib/datadog/ci/remote/component.rb
 - lib/datadog/ci/remote/library_settings.rb
 - lib/datadog/ci/remote/library_settings_client.rb
+- lib/datadog/ci/remote/slow_test_retries.rb
 - lib/datadog/ci/span.rb
 - lib/datadog/ci/test.rb
 - lib/datadog/ci/test_module.rb
@@ -150,10 +151,16 @@ files:
 - lib/datadog/ci/test_optimisation/skippable.rb
 - lib/datadog/ci/test_optimisation/telemetry.rb
 - lib/datadog/ci/test_retries/component.rb
+- lib/datadog/ci/test_retries/driver/base.rb
+- lib/datadog/ci/test_retries/driver/no_retry.rb
+- lib/datadog/ci/test_retries/driver/retry_failed.rb
+- lib/datadog/ci/test_retries/driver/retry_new.rb
 - lib/datadog/ci/test_retries/null_component.rb
 - lib/datadog/ci/test_retries/strategy/base.rb
 - lib/datadog/ci/test_retries/strategy/no_retry.rb
 - lib/datadog/ci/test_retries/strategy/retry_failed.rb
+- lib/datadog/ci/test_retries/strategy/retry_new.rb
+- lib/datadog/ci/test_retries/unique_tests_client.rb
 - lib/datadog/ci/test_session.rb
 - lib/datadog/ci/test_suite.rb
 - lib/datadog/ci/test_visibility/component.rb
@@ -187,7 +194,6 @@ files:
 - lib/datadog/ci/utils/bundle.rb
 - lib/datadog/ci/utils/configuration.rb
 - lib/datadog/ci/utils/git.rb
-- lib/datadog/ci/utils/identity.rb
 - lib/datadog/ci/utils/parsing.rb
 - lib/datadog/ci/utils/telemetry.rb
 - lib/datadog/ci/utils/test_run.rb
@@ -219,7 +225,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 2.0.0
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.16
 signing_key: 
 specification_version: 4
 summary: Datadog CI visibility for your ruby application

data/lib/datadog/ci/contrib/cucumber/configuration_override.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "formatter"
-
-module Datadog
-  module CI
-    module Contrib
-      module Cucumber
-        # Changes behaviour of Cucumber::Configuration class
-        module ConfigurationOverride
-          def self.included(base)
-            base.prepend(InstanceMethods)
-          end
-
-          # Instance methods for configuration
-          module InstanceMethods
-            def retry_attempts
-              super if !datadog_test_retries_component&.retry_failed_tests_enabled
-
-              datadog_test_retries_component&.retry_failed_tests_max_attempts
-            end
-
-            def retry_total_tests
-              super if !datadog_test_retries_component&.retry_failed_tests_enabled
-
-              datadog_test_retries_component&.retry_failed_tests_total_limit
-            end
-
-            def datadog_test_retries_component
-              Datadog.send(:components).test_retries
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/datadog/ci/utils/identity.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module Datadog
-  module CI
-    module Utils
-      module Identity
-        def self.included(base)
-          base.singleton_class.prepend(ClassMethods)
-        end
-
-        module ClassMethods
-          # return datadog-ci gem version instead of datadog gem version
-          def gem_datadog_version
-            Datadog::CI::VERSION::STRING
-          end
-        end
-      end
-    end
-  end
-end