datadog-ci 1.15.0 → 1.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4507da31359cd997cb38e73190a4e6514ad8c69d86f9ce1b0c47fd7764e5dbd
-  data.tar.gz: 515872f426f95492fed9d1577f14bb7b1e447b0c011edf2bbfbb7ea23992d9cc
+  metadata.gz: 4f7cf7ed07d7a4543498333996a19347f64010eef7900f53d5790ad6e4fe2aaf
+  data.tar.gz: 7b3ebdaaf30c557ac23f3720f72ef56ab1b9c1f9686e8f62dc597e3c938cfe35
 SHA512:
-  metadata.gz: 3d0db7cb6a13a4c2f0b38b22c8dc2470a9ec760c38312c929a7b6224b118e18b279addf76901f7f094f60123a9e65d9afef06c15970bf01b247bc031bf80725d
-  data.tar.gz: f731bc7979a69ae63362d06d0313754bd825a22baf934cb682c1dc2a16a8584d0c3ece806f2bfb75d42924c09fff6296bf8c5c1be142afbe3ab7716bcb753815
+  metadata.gz: 9412f518c4bba62d062448950f10b956abebe22e33680341454f60f67fd48c25b155d9e0ed3f7d378bcb33f21ee0fa586bee2ff03f3a2ad3f4b753a8a24bb36c
+  data.tar.gz: eb8710ab135550d9c26e1742aefb5ebb05b2c8a1f91e9175e44b0181c4934f1ff5453749820b8720e1febe5c79502e2489d46442f80c4836e56a394c931f7143

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
 
+## [1.16.0] - 2025-04-15
+
+
+### Added
+
+* agentless logs submission for tests ([#306][])
+* lograge support for agentless logs feature ([#308][])
+* semantic_logger instrumentation for agentless logs feature ([#311][])
+* retry reason "external" ([#312][])
+
 ## [1.15.0] - 2025-03-25
 
 
@@ -417,7 +427,8 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 
 - Ruby versions < 2.7 no longer supported ([#8][])
 
-[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.15.0...main
+[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.16.0...main
+[1.16.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.15.0...v1.16.0
 [1.15.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.14.0...v1.15.0
 [1.14.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.13.0...v1.14.0
 [1.13.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.12.0...v1.13.0
@@ -597,4 +608,8 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 [#298]: https://github.com/DataDog/datadog-ci-rb/issues/298
 [#299]: https://github.com/DataDog/datadog-ci-rb/issues/299
 [#300]: https://github.com/DataDog/datadog-ci-rb/issues/300
-[#301]: https://github.com/DataDog/datadog-ci-rb/issues/301
+[#301]: https://github.com/DataDog/datadog-ci-rb/issues/301
+[#306]: https://github.com/DataDog/datadog-ci-rb/issues/306
+[#308]: https://github.com/DataDog/datadog-ci-rb/issues/308
+[#311]: https://github.com/DataDog/datadog-ci-rb/issues/311
+[#312]: https://github.com/DataDog/datadog-ci-rb/issues/312

data/lib/datadog/ci/async_writer.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "datadog/core/workers/async"
+require "datadog/core/workers/queue"
+require "datadog/core/workers/polling"
+
+require "datadog/core/buffer/cruby"
+require "datadog/core/buffer/thread_safe"
+
+require "datadog/core/environment/ext"
+
+module Datadog
+  module CI
+    class AsyncWriter
+      include Core::Workers::Queue
+      include Core::Workers::Polling
+
+      attr_reader :transport
+
+      DEFAULT_BUFFER_MAX_SIZE = 10_000
+      DEFAULT_SHUTDOWN_TIMEOUT = 60
+
+      DEFAULT_INTERVAL = 3
+
+      def initialize(transport:, options: {})
+        @transport = transport
+
+        # Workers::Polling settings
+        self.enabled = options.fetch(:enabled, true)
+
+        # Workers::Async::Thread settings
+        self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_RESTART
+
+        # Workers::IntervalLoop settings
+        self.loop_base_interval = options[:interval] || DEFAULT_INTERVAL
+        self.loop_back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
+        self.loop_back_off_max = options[:back_off_max] if options.key?(:back_off_max)
+
+        @buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
+
+        self.buffer = buffer_klass.new(@buffer_size)
+
+        @shutdown_timeout = options.fetch(:shutdown_timeout, DEFAULT_SHUTDOWN_TIMEOUT)
+
+        @stopped = false
+      end
+
+      def write(event)
+        return if @stopped
+
+        # Start worker thread. If the process has forked, it will trigger #after_fork to
+        # reconfigure the worker accordingly.
+        perform
+
+        enqueue(event)
+      end
+
+      def perform(*events)
+        responses = transport.send_events(events)
+
+        if responses.find(&:server_error?)
+          loop_back_off!
+          Datadog.logger.warn { "Encountered server error while sending events: #{responses}" }
+        end
+
+        nil
+      rescue => e
+        Datadog.logger.warn { "Error while sending events: #{e}" }
+        loop_back_off!
+      end
+
+      def stop(force_stop = false, timeout = @shutdown_timeout)
+        @stopped = true
+
+        buffer.close if running?
+
+        super
+      end
+
+      def enqueue(event)
+        buffer.push(event)
+      end
+
+      def dequeue
+        buffer.pop
+      end
+
+      def work_pending?
+        !buffer.empty?
+      end
+
+      def async?
+        true
+      end
+
+      def after_fork
+        # In multiprocess environments, forks will share the same buffer until its written to.
+        # A.K.A. copy-on-write. We don't want forks to write events generated from another process.
+        # Instead, we reset it after the fork. (Make sure any enqueue operations happen after this.)
+        self.buffer = buffer_klass.new(@buffer_size)
+      end
+
+      def buffer_klass
+        if Core::Environment::Ext::RUBY_ENGINE == "ruby"
+          Core::Buffer::CRuby
+        else
+          Core::Buffer::ThreadSafe
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/configuration/components.rb

@@ -4,6 +4,8 @@ require "datadog/core/telemetry/ext"
 
 require_relative "../ext/settings"
 require_relative "../git/tree_uploader"
+require_relative "../logs/component"
+require_relative "../logs/transport"
 require_relative "../remote/component"
 require_relative "../remote/library_settings_client"
 require_relative "../test_management/component"
@@ -11,7 +13,6 @@ require_relative "../test_management/null_component"
 require_relative "../test_management/tests_properties"
 require_relative "../test_optimisation/component"
 require_relative "../test_optimisation/coverage/transport"
-require_relative "../test_optimisation/coverage/writer"
 require_relative "../test_retries/component"
 require_relative "../test_retries/null_component"
 require_relative "../test_visibility/component"
@@ -20,12 +21,13 @@ require_relative "../test_visibility/known_tests"
 require_relative "../test_visibility/null_component"
 require_relative "../test_visibility/serializers/factories/test_level"
 require_relative "../test_visibility/serializers/factories/test_suite_level"
+require_relative "../test_visibility/null_transport"
 require_relative "../test_visibility/transport"
 require_relative "../transport/adapters/telemetry_webmock_safe_adapter"
-require_relative "../test_visibility/null_transport"
 require_relative "../transport/api/builder"
 require_relative "../utils/parsing"
 require_relative "../utils/test_run"
+require_relative "../async_writer"
 require_relative "../worker"
 
 module Datadog
@@ -33,7 +35,8 @@ module Datadog
     module Configuration
       # Adds CI behavior to Datadog trace components
       module Components
-        attr_reader :test_visibility, :test_optimisation, :git_tree_upload_worker, :ci_remote, :test_retries, :test_management
+        attr_reader :test_visibility, :test_optimisation, :git_tree_upload_worker, :ci_remote, :test_retries,
+          :test_management, :agentless_logs_submission
 
         def initialize(settings)
           @test_optimisation = nil
@@ -56,6 +59,7 @@ module Datadog
 
           @test_visibility&.shutdown!
           @test_optimisation&.shutdown!
+          @agentless_logs_submission&.shutdown!
           @git_tree_upload_worker&.stop
         end
 
@@ -132,6 +136,8 @@ module Datadog
             known_tests_client: build_known_tests_client(settings, test_visibility_api),
             context_service_uri: settings.ci.test_visibility_drb_server_uri
           )
+
+          @agentless_logs_submission = build_agentless_logs_component(settings, test_visibility_api)
         end
 
         def build_test_optimisation(settings, test_visibility_api)
@@ -224,7 +230,7 @@ module Datadog
           # nil means that coverage event will be ignored
           return nil if api.nil? || settings.ci.discard_traces
 
-          TestOptimisation::Coverage::Writer.new(
+          AsyncWriter.new(
             transport: TestOptimisation::Coverage::Transport.new(api: api)
           )
         end
@@ -256,6 +262,29 @@ module Datadog
           )
         end
 
+        def build_agentless_logs_component(settings, api)
+          if settings.ci.agentless_logs_submission_enabled && !settings.ci.agentless_mode_enabled
+            Datadog.logger.warn(
+              "Agentless logs submission is enabled but agentless mode is not enabled. " \
+              "Logs will not be submitted. " \
+              "Please make sure to set DD_CIVISIBILITY_AGENTLESS_ENABLED to true if you want to submit logs in agentless mode. " \
+              "Otherwise, set DD_AGENTLESS_LOG_SUBMISSION_ENABLED to 0 and use Datadog Agent to submit logs."
+            )
+            settings.ci.agentless_logs_submission_enabled = false
+          end
+
+          Logs::Component.new(
+            enabled: settings.ci.agentless_logs_submission_enabled,
+            writer: build_logs_writer(settings, api)
+          )
+        end
+
+        def build_logs_writer(settings, api)
+          return nil if api.nil? || settings.ci.discard_traces
+
+          AsyncWriter.new(transport: Logs::Transport.new(api: api), options: {buffer_size: 1024})
+        end
+
         # fetch custom tags provided by the user in DD_TAGS env var
         # with prefix test.configuration.
         def custom_configuration(settings)
@@ -298,7 +327,7 @@ module Datadog
 
             # patch gem's telemetry transport layer to use Net::HTTP instead of WebMock's Net::HTTP
             Core::Telemetry::Http::Adapters::Net.include(CI::Transport::Adapters::TelemetryWebmockSafeAdapter)
-          rescue => e
+          rescue LoadError, StandardError => e
             Datadog.logger.warn("Failed to patch Datadog gem's telemetry layer: #{e}")
           end
         end

data/lib/datadog/ci/configuration/settings.rb

@@ -128,6 +128,17 @@ module Datadog
                 o.default 20
               end
 
+              option :agentless_logs_submission_enabled do |o|
+                o.type :bool
+                o.env CI::Ext::Settings::ENV_AGENTLESS_LOGS_SUBMISSION_ENABLED
+                o.default false
+              end
+
+              option :agentless_logs_submission_url do |o|
+                o.type :string, nilable: true
+                o.env CI::Ext::Settings::ENV_AGENTLESS_LOGS_SUBMISSION_URL
+              end
+
               # internal only
               option :discard_traces do |o|
                 o.type :bool

data/lib/datadog/ci/contrib/activesupport/configuration/settings.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "../ext"
+require_relative "../../settings"
+require_relative "../../../utils/configuration"
+
+module Datadog
+  module CI
+    module Contrib
+      module ActiveSupport
+        module Configuration
+          # Custom settings for the ActiveSupport integration
+          # @public_api
+          class Settings < Datadog::CI::Contrib::Settings
+            option :enabled do |o|
+              o.type :bool
+              o.env Ext::ENV_ENABLED
+              o.default true
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/activesupport/ext.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module ActiveSupport
+        # Datadog ActiveSupport integration constants
+        module Ext
+          ENV_ENABLED = "DD_CI_ACTIVESUPPORT_ENABLED"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/activesupport/integration.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "../integration"
+require_relative "configuration/settings"
+require_relative "patcher"
+
+module Datadog
+  module CI
+    module Contrib
+      module ActiveSupport
+        # Description of ActiveSupport integration
+        class Integration < Datadog::CI::Contrib::Integration
+          MINIMUM_VERSION = Gem::Version.new("5.0")
+
+          def version
+            Gem.loaded_specs["activesupport"]&.version
+          end
+
+          def loaded?
+            !defined?(::Rails).nil? && !defined?(::ActiveSupport).nil? && !defined?(::ActiveSupport::TaggedLogging).nil? &&
+              !defined?(::ActiveSupport::TaggedLogging::Formatter).nil?
+          end
+
+          def compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          def late_instrument?
+            true
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/activesupport/logs_formatter.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module ActiveSupport
+        module LogsFormatter
+          def call(severity, timestamp, progname, msg)
+            # don't even construct an object for every log message if agentless logs submission is not enabled
+            return super unless datadog_logs_component.enabled
+            return super unless datadog_configuration[:enabled]
+
+            # additional precaution because we cannot use targeted prepend in Ruby 2.7, so method :tags_text might
+            # not be available (highly unlikely, but not unimaginable)
+            #
+            # (see Datadog::CI::Contrib::ActiveSupport::Patcher for explanation)
+            return super unless respond_to?(:tags_text)
+
+            message = "#{msg} #{tags_text}"
+            return super unless message.include?("dd.trace_id")
+
+            datadog_logs_component.write({
+              message: message,
+              level: severity
+            })
+
+            super
+          end
+
+          def datadog_logs_component
+            Datadog.send(:components).agentless_logs_submission
+          end
+
+          def datadog_configuration
+            Datadog.configuration.ci[:activesupport]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/activesupport/patcher.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative "../patcher"
+require_relative "logs_formatter"
+
+module Datadog
+  module CI
+    module Contrib
+      module ActiveSupport
+        # Patcher enables patching of activesupport module
+        module Patcher
+          include Datadog::CI::Contrib::Patcher
+
+          module_function
+
+          def patch
+            unless datadog_logs_component.enabled
+              Datadog.logger.debug("Datadog logs submission is disabled, skipping activesupport patching")
+              return
+            end
+
+            unless ::Rails.logger.formatter.is_a?(::ActiveSupport::TaggedLogging::Formatter)
+              Datadog.logger.debug {
+                "Rails logger formatter is not an instance of ActiveSupport::TaggedLogging::Formatter, skipping activesupport patching. " \
+                "Formatter: #{::Rails.logger.formatter.class}"
+              }
+              return
+            end
+
+            # Before Ruby 3.0, prepending to a module did not change existing instances where this module was included
+            # It means that for Ruby 2.7 we have to patch formatter's class directly
+            #
+            # Context:
+            # - https://bugs.ruby-lang.org/issues/9573
+            # - https://rubyreferences.github.io/rubychanges/3.0.html#include-and-prepend-now-affects-modules-including-the-receiver
+            if RUBY_VERSION.start_with?("2.7")
+              Rails.logger.formatter.class.prepend(LogsFormatter)
+            else
+              ::ActiveSupport::TaggedLogging::Formatter.prepend(LogsFormatter)
+            end
+          end
+
+          def datadog_logs_component
+            Datadog.send(:components).agentless_logs_submission
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/lograge/configuration/settings.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "../ext"
+require_relative "../../settings"
+require_relative "../../../utils/configuration"
+
+module Datadog
+  module CI
+    module Contrib
+      module Lograge
+        module Configuration
+          # Custom settings for the Lograge integration
+          # @public_api
+          class Settings < Datadog::CI::Contrib::Settings
+            option :enabled do |o|
+              o.type :bool
+              o.env Ext::ENV_ENABLED
+              o.default true
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/lograge/ext.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module Lograge
+        # Datadog Lograge integration constants
+        module Ext
+          ENV_ENABLED = "DD_CI_LOGRAGE_ENABLED"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/lograge/integration.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "../integration"
+require_relative "configuration/settings"
+require_relative "patcher"
+
+module Datadog
+  module CI
+    module Contrib
+      module Lograge
+        # Description of Lograge integration
+        class Integration < Datadog::CI::Contrib::Integration
+          MINIMUM_VERSION = Gem::Version.new("0.14.0")
+
+          def version
+            Gem.loaded_specs["lograge"]&.version
+          end
+
+          def loaded?
+            !defined?(::Lograge).nil? && !defined?(::Lograge::LogSubscribers).nil? &&
+              !defined?(::Lograge::LogSubscribers::Base).nil?
+          end
+
+          def compatible?
+            super && version && version >= MINIMUM_VERSION
+          end
+
+          def late_instrument?
+            true
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/lograge/log_subscriber.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module Lograge
+        module LogSubscriber
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          module InstanceMethods
+            private
+
+            def before_format(data, payload)
+              return super unless datadog_logs_component.enabled
+              return super unless datadog_configuration[:enabled]
+
+              result = super
+
+              if result.fetch(:dd, {}).fetch(:trace_id, nil).nil?
+                Datadog.logger.debug { "Discarding uncorrelated log event: #{result.inspect}" }
+                return result
+              end
+              datadog_logs_component.write(result)
+              result
+            end
+
+            def datadog_logs_component
+              Datadog.send(:components).agentless_logs_submission
+            end
+
+            def datadog_configuration
+              Datadog.configuration.ci[:lograge]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/lograge/patcher.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative "../patcher"
+require_relative "log_subscriber"
+
+module Datadog
+  module CI
+    module Contrib
+      module Lograge
+        # Patcher enables patching of lograge module
+        module Patcher
+          include Datadog::CI::Contrib::Patcher
+
+          module_function
+
+          def patch
+            unless datadog_logs_component.enabled
+              Datadog.logger.debug("Datadog logs submission is disabled, skipping lograge patching")
+              return
+            end
+
+            ::Lograge::LogSubscribers::Base.include(LogSubscriber)
+          end
+
+          def datadog_logs_component
+            Datadog.send(:components).agentless_logs_submission
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/parallel_tests/patcher.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative "cli"
-require_relative "integration"
 require_relative "../patcher"
 
 module Datadog

data/lib/datadog/ci/contrib/semantic_logger/configuration/settings.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "../ext"
+require_relative "../../settings"
+require_relative "../../../utils/configuration"
+
+module Datadog
+  module CI
+    module Contrib
+      module SemanticLogger
+        module Configuration
+          # Custom settings for the SemanticLogger integration
+          # @public_api
+          class Settings < Datadog::CI::Contrib::Settings
+            option :enabled do |o|
+              o.type :bool
+              o.env Ext::ENV_ENABLED
+              o.default true
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/semantic_logger/ext.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module SemanticLogger
+        # Datadog SemanticLogger integration constants
+        module Ext
+          ENV_ENABLED = "DD_CI_SEMANTIC_LOGGER_ENABLED"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/semantic_logger/integration.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "../integration"
+require_relative "configuration/settings"
+require_relative "patcher"
+
+module Datadog
+  module CI
+    module Contrib
+      module SemanticLogger
+        # Description of SemanticLogger integration
+        class Integration < Datadog::CI::Contrib::Integration
+          MINIMUM_VERSION = Gem::Version.new("4.0")
+
+          def version
+            Gem.loaded_specs["semantic_logger"]&.version
+          end
+
+          def loaded?
+            !defined?(::SemanticLogger).nil? && !defined?(::SemanticLogger::Logger).nil?
+          end
+
+          def compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          def late_instrument?
+            true
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/semantic_logger/logger.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module SemanticLogger
+        module Logger
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          module InstanceMethods
+            def log(log, message = nil, progname = nil, &block)
+              return super unless log.is_a?(::SemanticLogger::Log)
+              return super unless datadog_logs_component.enabled
+
+              result = super
+
+              datadog_logs_component.write(log.to_h.clone)
+
+              result
+            end
+
+            def datadog_logs_component
+              Datadog.send(:components).agentless_logs_submission
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/semantic_logger/patcher.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative "../patcher"
+require_relative "logger"
+
+module Datadog
+  module CI
+    module Contrib
+      module SemanticLogger
+        # Patcher enables patching of semantic_logger module
+        module Patcher
+          include Datadog::CI::Contrib::Patcher
+
+          module_function
+
+          def patch
+            unless datadog_logs_component.enabled
+              Datadog.logger.debug("Datadog logs submission is disabled, skipping semantic_logger patching")
+              return
+            end
+
+            ::SemanticLogger::Logger.include(Logger)
+          end
+
+          def datadog_logs_component
+            Datadog.send(:components).agentless_logs_submission
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/ext/settings.rb

@@ -23,6 +23,8 @@ module Datadog
         ENV_TEST_MANAGEMENT_ENABLED = "DD_TEST_MANAGEMENT_ENABLED"
         ENV_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES = "DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES"
         ENV_TEST_VISIBILITY_DRB_SERVER_URI = "DD_TEST_VISIBILITY_DRB_SERVER_URI"
+        ENV_AGENTLESS_LOGS_SUBMISSION_ENABLED = "DD_AGENTLESS_LOG_SUBMISSION_ENABLED"
+        ENV_AGENTLESS_LOGS_SUBMISSION_URL = "DD_AGENTLESS_LOG_SUBMISSION_URL"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = %w[

data/lib/datadog/ci/ext/test.rb

@@ -151,9 +151,10 @@ module Datadog
 
         # possible reasons why a test was retried
         module RetryReason
-          RETRY_NEW = "efd"
-          RETRY_FAILED = "atr"
+          RETRY_DETECT_FLAKY = "early_flake_detection"
+          RETRY_FAILED = "auto_test_retries"
           RETRY_FLAKY_FIXED = "attempt_to_fix"
+          RETRY_EXTERNAL = "external"
         end
 
         # possible reasons why a test was skipped

data/lib/datadog/ci/ext/transport.rb

@@ -28,6 +28,8 @@ module Datadog
         TEST_COVERAGE_INTAKE_HOST_PREFIX = "citestcov-intake"
         TEST_COVERAGE_INTAKE_PATH = "/api/v2/citestcov"
 
+        LOGS_INTAKE_HOST_PREFIX = "http-intake.logs"
+
         DD_API_HOST_PREFIX = "api"
 
         DD_API_SETTINGS_PATH = "/api/v2/libraries/tests/services/setting"

data/lib/datadog/ci/logs/component.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "datadog/core/environment/platform"
+
+module Datadog
+  module CI
+    module Logs
+      class Component
+        attr_reader :enabled
+
+        def initialize(enabled:, writer:)
+          @enabled = enabled && !writer.nil?
+          @writer = writer
+        end
+
+        def write(event)
+          return unless enabled
+
+          add_common_tags!(event)
+          @writer&.write(event)
+
+          nil
+        end
+
+        def shutdown!
+          @writer&.stop
+        end
+
+        private
+
+        def add_common_tags!(event)
+          test_session = test_visibility.active_test_session
+
+          event[:ddsource] ||= "ruby"
+          event[:ddtags] ||= "datadog.product:citest"
+          event[:service] ||= test_session&.service
+          event[:hostname] ||= Datadog::Core::Environment::Platform.hostname
+        end
+
+        def test_visibility
+          ::Datadog.send(:components).test_visibility
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/logs/transport.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "json"
+
+require "datadog/core/chunker"
+
+module Datadog
+  module CI
+    module Logs
+      class Transport
+        DEFAULT_MAX_PAYLOAD_SIZE = 4.5 * 1024 * 1024
+
+        attr_reader :api,
+          :max_payload_size
+
+        def initialize(api:, max_payload_size: DEFAULT_MAX_PAYLOAD_SIZE)
+          @api = api
+          @max_payload_size = max_payload_size
+        end
+
+        def send_events(events)
+          return [] if events.nil? || events.empty?
+
+          Datadog.logger.debug { "[#{self.class.name}] Sending #{events.count} events..." }
+
+          encoded_events = events.filter_map do |event|
+            encoded_event = event.to_json
+            if event_too_large?(event, encoded_event)
+              next
+            end
+
+            encoded_event
+          end
+
+          responses = []
+          Datadog::Core::Chunker.chunk_by_size(encoded_events, max_payload_size).map do |chunk|
+            encoded_payload = pack_events(chunk)
+            Datadog.logger.debug do
+              "[#{self.class.name}] Send chunk of #{chunk.count} events; payload size #{encoded_payload.size}"
+            end
+
+            response = send_payload(encoded_payload)
+
+            responses << response
+          end
+
+          responses
+        end
+
+        private
+
+        def pack_events(encoded_events)
+          "[#{encoded_events.join(",")}]"
+        end
+
+        def event_too_large?(event, encoded_event)
+          return false unless encoded_event.size > max_payload_size
+
+          # This single event is too large, we can't flush it
+          Datadog.logger.debug(
+            "[#{self.class.name}] Dropping event for logs intake. Payload too large: '#{event.inspect}'"
+          )
+
+          true
+        end
+
+        def send_payload(encoded_payload)
+          @api.logs_intake_request(path: "/v1/input", payload: encoded_payload)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test.rb

@@ -25,6 +25,10 @@ module Datadog
       # Finishes the current test.
       # @return [void]
       def finish
+        if is_retry? && retry_reason.nil?
+          set_tag(Ext::Test::TAG_RETRY_REASON, Ext::Test::RetryReason::RETRY_EXTERNAL)
+        end
+
         test_visibility.deactivate_test
 
         super
@@ -68,6 +72,12 @@ module Datadog
         get_tag(Ext::Test::TAG_IS_RETRY) == "true"
       end
 
+      # Returns string with a reason why test was retried
+      # @return [String] retry reason
+      def retry_reason
+        get_tag(Ext::Test::TAG_RETRY_REASON)
+      end
+
       # Returns "true" if this span represents a test that wasn't known to Datadog before.
       # @return [Boolean] true if this test is a new one, false otherwise.
       def is_new?

data/lib/datadog/ci/test_retries/driver/retry_new.rb

@@ -36,7 +36,7 @@ module Datadog
           end
 
           def retry_reason
-            Ext::Test::RetryReason::RETRY_NEW
+            Ext::Test::RetryReason::RETRY_DETECT_FLAKY
           end
         end
       end

data/lib/datadog/ci/test_visibility/null_component.rb

@@ -8,7 +8,9 @@ module Datadog
         def configure(_, _)
         end
 
-        def start_test_session(service: nil, tags: {}, estimated_total_tests_count: 0)
+        def start_test_session(
+          service: nil, tags: {}, estimated_total_tests_count: 0, distributed: false, local_test_suites_mode: true
+        )
           skip_tracing
         end
 

data/lib/datadog/ci/transport/api/agentless.rb

@@ -10,11 +10,12 @@ module Datadog
         class Agentless < Base
           attr_reader :api_key
 
-          def initialize(api_key:, citestcycle_url:, api_url:, citestcov_url:)
+          def initialize(api_key:, citestcycle_url:, api_url:, citestcov_url:, logs_intake_url:)
             @api_key = api_key
             @citestcycle_http = build_http_client(citestcycle_url, compress: true)
             @api_http = build_http_client(api_url, compress: false)
             @citestcov_http = build_http_client(citestcov_url, compress: true)
+            @logs_intake_http = build_http_client(logs_intake_url, compress: true)
           end
 
           def citestcycle_request(path:, payload:, headers: {}, verb: "post")
@@ -42,6 +43,12 @@ module Datadog
             perform_request(@citestcov_http, path: path, payload: @citestcov_payload, headers: headers, verb: verb)
           end
 
+          def logs_intake_request(path:, payload:, headers: {}, verb: "post")
+            super
+
+            perform_request(@logs_intake_http, path: path, payload: payload, headers: headers, verb: verb)
+          end
+
           private
 
           def perform_request(http_client, path:, payload:, headers:, verb:, accept_compressed_response: false)

data/lib/datadog/ci/transport/api/base.rb

@@ -38,6 +38,10 @@ module Datadog
             ].join("\r\n")
           end
 
+          def logs_intake_request(path:, payload:, headers: {}, verb: "post")
+            headers[Ext::Transport::HEADER_CONTENT_TYPE] ||= Ext::Transport::CONTENT_TYPE_JSON
+          end
+
           def headers_with_default(headers)
             request_headers = default_headers
             request_headers.merge!(headers)

data/lib/datadog/ci/transport/api/builder.rb

@@ -27,11 +27,15 @@ module Datadog
             citestcov_url = settings.ci.agentless_url ||
               "https://#{Ext::Transport::TEST_COVERAGE_INTAKE_HOST_PREFIX}.#{dd_site}:443"
 
+            logs_intake_url = settings.ci.agentless_url ||
+              "https://#{Ext::Transport::LOGS_INTAKE_HOST_PREFIX}.#{dd_site}:443"
+
             Agentless.new(
               api_key: settings.api_key,
               citestcycle_url: citestcycle_url,
               api_url: api_url,
-              citestcov_url: citestcov_url
+              citestcov_url: citestcov_url,
+              logs_intake_url: logs_intake_url
             )
           end
 

data/lib/datadog/ci/transport/api/evp_proxy.rb

@@ -46,6 +46,10 @@ module Datadog
             perform_request(@agent_intake_http, path: path, payload: @citestcov_payload, headers: headers, verb: verb)
           end
 
+          def logs_intake_request(path:, payload:, headers: {}, verb: "post")
+            raise NotImplementedError, "Logs intake is not supported in EVP proxy mode"
+          end
+
           private
 
           def perform_request(http_client, path:, payload:, headers:, verb:)

data/lib/datadog/ci/version.rb

@@ -4,7 +4,7 @@ module Datadog
   module CI
     module VERSION
       MAJOR = 1
-      MINOR = 15
+      MINOR = 16
       PATCH = 0
       PRE = nil
       BUILD = nil

data/lib/datadog/ci.rb

@@ -429,6 +429,9 @@ require_relative "ci/contrib/parallel_tests/integration"
 require_relative "ci/contrib/selenium/integration"
 require_relative "ci/contrib/cuprite/integration"
 require_relative "ci/contrib/simplecov/integration"
+require_relative "ci/contrib/activesupport/integration"
+require_relative "ci/contrib/lograge/integration"
+require_relative "ci/contrib/semantic_logger/integration"
 
 # Configuration extensions
 require_relative "ci/configuration/extensions"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: datadog-ci
 version: !ruby/object:Gem::Version
-  version: 1.15.0
+  version: 1.16.0
 platform: ruby
 authors:
 - Datadog, Inc.
 bindir: exe
 cert_chain: []
-date: 2025-03-25 00:00:00.000000000 Z
+date: 2025-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: datadog
@@ -74,6 +74,7 @@ files:
 - ext/datadog_cov/datadog_cov.c
 - ext/datadog_cov/extconf.rb
 - lib/datadog/ci.rb
+- lib/datadog/ci/async_writer.rb
 - lib/datadog/ci/auto_instrument.rb
 - lib/datadog/ci/cli/cli.rb
 - lib/datadog/ci/cli/command/base.rb
@@ -87,6 +88,11 @@ files:
 - lib/datadog/ci/configuration/components.rb
 - lib/datadog/ci/configuration/extensions.rb
 - lib/datadog/ci/configuration/settings.rb
+- lib/datadog/ci/contrib/activesupport/configuration/settings.rb
+- lib/datadog/ci/contrib/activesupport/ext.rb
+- lib/datadog/ci/contrib/activesupport/integration.rb
+- lib/datadog/ci/contrib/activesupport/logs_formatter.rb
+- lib/datadog/ci/contrib/activesupport/patcher.rb
 - lib/datadog/ci/contrib/ciqueue/integration.rb
 - lib/datadog/ci/contrib/ciqueue/patcher.rb
 - lib/datadog/ci/contrib/cucumber/configuration/settings.rb
@@ -108,6 +114,11 @@ files:
 - lib/datadog/ci/contrib/knapsack/integration.rb
 - lib/datadog/ci/contrib/knapsack/patcher.rb
 - lib/datadog/ci/contrib/knapsack/runner.rb
+- lib/datadog/ci/contrib/lograge/configuration/settings.rb
+- lib/datadog/ci/contrib/lograge/ext.rb
+- lib/datadog/ci/contrib/lograge/integration.rb
+- lib/datadog/ci/contrib/lograge/log_subscriber.rb
+- lib/datadog/ci/contrib/lograge/patcher.rb
 - lib/datadog/ci/contrib/minitest/configuration/settings.rb
 - lib/datadog/ci/contrib/minitest/ext.rb
 - lib/datadog/ci/contrib/minitest/helpers.rb
@@ -138,6 +149,11 @@ files:
 - lib/datadog/ci/contrib/selenium/integration.rb
 - lib/datadog/ci/contrib/selenium/navigation.rb
 - lib/datadog/ci/contrib/selenium/patcher.rb
+- lib/datadog/ci/contrib/semantic_logger/configuration/settings.rb
+- lib/datadog/ci/contrib/semantic_logger/ext.rb
+- lib/datadog/ci/contrib/semantic_logger/integration.rb
+- lib/datadog/ci/contrib/semantic_logger/logger.rb
+- lib/datadog/ci/contrib/semantic_logger/patcher.rb
 - lib/datadog/ci/contrib/settings.rb
 - lib/datadog/ci/contrib/simplecov/configuration/settings.rb
 - lib/datadog/ci/contrib/simplecov/ext.rb
@@ -178,6 +194,8 @@ files:
 - lib/datadog/ci/git/tree_uploader.rb
 - lib/datadog/ci/git/upload_packfile.rb
 - lib/datadog/ci/git/user.rb
+- lib/datadog/ci/logs/component.rb
+- lib/datadog/ci/logs/transport.rb
 - lib/datadog/ci/readonly_test_module.rb
 - lib/datadog/ci/readonly_test_session.rb
 - lib/datadog/ci/remote/component.rb
@@ -194,7 +212,6 @@ files:
 - lib/datadog/ci/test_optimisation/coverage/ddcov.rb
 - lib/datadog/ci/test_optimisation/coverage/event.rb
 - lib/datadog/ci/test_optimisation/coverage/transport.rb
-- lib/datadog/ci/test_optimisation/coverage/writer.rb
 - lib/datadog/ci/test_optimisation/skippable.rb
 - lib/datadog/ci/test_optimisation/skippable_percentage/base.rb
 - lib/datadog/ci/test_optimisation/skippable_percentage/calculator.rb

data/lib/datadog/ci/test_optimisation/coverage/writer.rb

@@ -1,116 +0,0 @@
-# frozen_string_literal: true
-
-require "datadog/core/workers/async"
-require "datadog/core/workers/queue"
-require "datadog/core/workers/polling"
-
-require "datadog/core/buffer/cruby"
-require "datadog/core/buffer/thread_safe"
-
-require "datadog/core/environment/ext"
-
-module Datadog
-  module CI
-    module TestOptimisation
-      module Coverage
-        class Writer
-          include Core::Workers::Queue
-          include Core::Workers::Polling
-
-          attr_reader :transport
-
-          DEFAULT_BUFFER_MAX_SIZE = 10_000
-          DEFAULT_SHUTDOWN_TIMEOUT = 60
-
-          DEFAULT_INTERVAL = 3
-
-          def initialize(transport:, options: {})
-            @transport = transport
-
-            # Workers::Polling settings
-            self.enabled = options.fetch(:enabled, true)
-
-            # Workers::Async::Thread settings
-            self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_RESTART
-
-            # Workers::IntervalLoop settings
-            self.loop_base_interval = options[:interval] || DEFAULT_INTERVAL
-            self.loop_back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
-            self.loop_back_off_max = options[:back_off_max] if options.key?(:back_off_max)
-
-            @buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
-
-            self.buffer = buffer_klass.new(@buffer_size)
-
-            @shutdown_timeout = options.fetch(:shutdown_timeout, DEFAULT_SHUTDOWN_TIMEOUT)
-
-            @stopped = false
-          end
-
-          def write(event)
-            return if @stopped
-
-            # Start worker thread. If the process has forked, it will trigger #after_fork to
-            # reconfigure the worker accordingly.
-            perform
-
-            enqueue(event)
-          end
-
-          def perform(*events)
-            responses = transport.send_events(events)
-
-            if responses.find(&:server_error?)
-              loop_back_off!
-              Datadog.logger.warn { "Encountered server error while sending coverage events" }
-            end
-
-            nil
-          rescue => e
-            Datadog.logger.warn { "Error while sending coverage events: #{e}" }
-            loop_back_off!
-          end
-
-          def stop(force_stop = false, timeout = @shutdown_timeout)
-            @stopped = true
-
-            buffer.close if running?
-
-            super
-          end
-
-          def enqueue(event)
-            buffer.push(event)
-          end
-
-          def dequeue
-            buffer.pop
-          end
-
-          def work_pending?
-            !buffer.empty?
-          end
-
-          def async?
-            true
-          end
-
-          def after_fork
-            # In multiprocess environments, forks will share the same buffer until its written to.
-            # A.K.A. copy-on-write. We don't want forks to write events generated from another process.
-            # Instead, we reset it after the fork. (Make sure any enqueue operations happen after this.)
-            self.buffer = buffer_klass.new(@buffer_size)
-          end
-
-          def buffer_klass
-            if Core::Environment::Ext::RUBY_ENGINE == "ruby"
-              Core::Buffer::CRuby
-            else
-              Core::Buffer::ThreadSafe
-            end
-          end
-        end
-      end
-    end
-  end
-end