datadog-ci 1.15.0 → 1.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4507da31359cd997cb38e73190a4e6514ad8c69d86f9ce1b0c47fd7764e5dbd
-  data.tar.gz: 515872f426f95492fed9d1577f14bb7b1e447b0c011edf2bbfbb7ea23992d9cc
+  metadata.gz: 1d2afc6148658bb41352ad4040c483406594852787e35aa523063a6ae452ec17
+  data.tar.gz: 3b7f9b07092e33b016b94b25d4aae2b12ff442f458dac6bea7b987adef4ca345
 SHA512:
-  metadata.gz: 3d0db7cb6a13a4c2f0b38b22c8dc2470a9ec760c38312c929a7b6224b118e18b279addf76901f7f094f60123a9e65d9afef06c15970bf01b247bc031bf80725d
-  data.tar.gz: f731bc7979a69ae63362d06d0313754bd825a22baf934cb682c1dc2a16a8584d0c3ece806f2bfb75d42924c09fff6296bf8c5c1be142afbe3ab7716bcb753815
+  metadata.gz: 7b368e3e3a1c6004307fe834f6f6442f51f7ab538434fb9a34ea42c6fbefa90f7995e54ed35aded0ddfc9e21ff34e70d91e845e9b82009a9e2977f3de9205ff6
+  data.tar.gz: c56a90e2d3fcc755f9fc224ea8e385c8393a8252487132ca4287377375ac57e839171853391ad9bcb4577a422b0b7c154f32f15c91b69b2b7b17de86aa396ca6

data/CHANGELOG.md

@@ -1,5 +1,31 @@
 ## [Unreleased]
 
+## [1.17.0] - 2025-05-19
+
+### Added
+
+* Impacted tests detection ([#318][])
+
+### Changed
+
+* flaky test management: add sha parameter to the tests_properties request ([#321][])
+* Send `attempt_to_fix_passed:false` when attempt_to_fix_validation did not succeed ([#320][])
+
+### Fixed
+
+* fix CODEOWNERS parsing issue where "**" wasn't matching zero folders ([#323][])
+* fix webmock compatibility of internal telemetry ([#309][])
+
+## [1.16.0] - 2025-04-15
+
+
+### Added
+
+* agentless logs submission for tests ([#306][])
+* lograge support for agentless logs feature ([#308][])
+* semantic_logger instrumentation for agentless logs feature ([#311][])
+* retry reason "external" ([#312][])
+
 ## [1.15.0] - 2025-03-25
 
 
@@ -417,7 +443,9 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 
 - Ruby versions < 2.7 no longer supported ([#8][])
 
-[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.15.0...main
+[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.17.0...main
+[1.17.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.16.0...v1.17.0
+[1.16.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.15.0...v1.16.0
 [1.15.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.14.0...v1.15.0
 [1.14.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.13.0...v1.14.0
 [1.13.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.12.0...v1.13.0
@@ -597,4 +625,13 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 [#298]: https://github.com/DataDog/datadog-ci-rb/issues/298
 [#299]: https://github.com/DataDog/datadog-ci-rb/issues/299
 [#300]: https://github.com/DataDog/datadog-ci-rb/issues/300
-[#301]: https://github.com/DataDog/datadog-ci-rb/issues/301
+[#301]: https://github.com/DataDog/datadog-ci-rb/issues/301
+[#306]: https://github.com/DataDog/datadog-ci-rb/issues/306
+[#308]: https://github.com/DataDog/datadog-ci-rb/issues/308
+[#309]: https://github.com/DataDog/datadog-ci-rb/issues/309
+[#311]: https://github.com/DataDog/datadog-ci-rb/issues/311
+[#312]: https://github.com/DataDog/datadog-ci-rb/issues/312
+[#318]: https://github.com/DataDog/datadog-ci-rb/issues/318
+[#320]: https://github.com/DataDog/datadog-ci-rb/issues/320
+[#321]: https://github.com/DataDog/datadog-ci-rb/issues/321
+[#323]: https://github.com/DataDog/datadog-ci-rb/issues/323

data/lib/datadog/ci/async_writer.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "datadog/core/workers/async"
+require "datadog/core/workers/queue"
+require "datadog/core/workers/polling"
+
+require "datadog/core/buffer/cruby"
+require "datadog/core/buffer/thread_safe"
+
+require "datadog/core/environment/ext"
+
+module Datadog
+  module CI
+    class AsyncWriter
+      include Core::Workers::Queue
+      include Core::Workers::Polling
+
+      attr_reader :transport
+
+      DEFAULT_BUFFER_MAX_SIZE = 10_000
+      DEFAULT_SHUTDOWN_TIMEOUT = 60
+
+      DEFAULT_INTERVAL = 3
+
+      def initialize(transport:, options: {})
+        @transport = transport
+
+        # Workers::Polling settings
+        self.enabled = options.fetch(:enabled, true)
+
+        # Workers::Async::Thread settings
+        self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_RESTART
+
+        # Workers::IntervalLoop settings
+        self.loop_base_interval = options[:interval] || DEFAULT_INTERVAL
+        self.loop_back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
+        self.loop_back_off_max = options[:back_off_max] if options.key?(:back_off_max)
+
+        @buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
+
+        self.buffer = buffer_klass.new(@buffer_size)
+
+        @shutdown_timeout = options.fetch(:shutdown_timeout, DEFAULT_SHUTDOWN_TIMEOUT)
+
+        @stopped = false
+      end
+
+      def write(event)
+        return if @stopped
+
+        # Start worker thread. If the process has forked, it will trigger #after_fork to
+        # reconfigure the worker accordingly.
+        perform
+
+        enqueue(event)
+      end
+
+      def perform(*events)
+        responses = transport.send_events(events)
+
+        if responses.find(&:server_error?)
+          loop_back_off!
+          Datadog.logger.warn { "Encountered server error while sending events: #{responses}" }
+        end
+
+        nil
+      rescue => e
+        Datadog.logger.warn { "Error while sending events: #{e}" }
+        loop_back_off!
+      end
+
+      def stop(force_stop = false, timeout = @shutdown_timeout)
+        @stopped = true
+
+        buffer.close if running?
+
+        super
+      end
+
+      def enqueue(event)
+        buffer.push(event)
+      end
+
+      def dequeue
+        buffer.pop
+      end
+
+      def work_pending?
+        !buffer.empty?
+      end
+
+      def async?
+        true
+      end
+
+      def after_fork
+        # In multiprocess environments, forks will share the same buffer until its written to.
+        # A.K.A. copy-on-write. We don't want forks to write events generated from another process.
+        # Instead, we reset it after the fork. (Make sure any enqueue operations happen after this.)
+        self.buffer = buffer_klass.new(@buffer_size)
+      end
+
+      def buffer_klass
+        if Core::Environment::Ext::RUBY_ENGINE == "ruby"
+          Core::Buffer::CRuby
+        else
+          Core::Buffer::ThreadSafe
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/codeowners/rule.rb

@@ -24,7 +24,12 @@ module Datadog
         private
 
         def flags
+          # prevents /path/* from matching subfolders
           return ::File::FNM_PATHNAME if pattern.end_with?("#{::File::SEPARATOR}*")
+          # prevents /path/*.rb from matching Ruby files in subfolders
+          return ::File::FNM_PATHNAME if pattern.include?("*.")
+          # allows /path/**/subfolder to match both /path/subfolder and /path/a/b/c/subfolder
+          return ::File::FNM_PATHNAME if pattern.include?("#{::File::SEPARATOR}**#{::File::SEPARATOR}")
           0
         end
       end

data/lib/datadog/ci/configuration/components.rb

@@ -4,6 +4,9 @@ require "datadog/core/telemetry/ext"
 
 require_relative "../ext/settings"
 require_relative "../git/tree_uploader"
+require_relative "../impacted_tests_detection/component"
+require_relative "../logs/component"
+require_relative "../logs/transport"
 require_relative "../remote/component"
 require_relative "../remote/library_settings_client"
 require_relative "../test_management/component"
@@ -11,7 +14,6 @@ require_relative "../test_management/null_component"
 require_relative "../test_management/tests_properties"
 require_relative "../test_optimisation/component"
 require_relative "../test_optimisation/coverage/transport"
-require_relative "../test_optimisation/coverage/writer"
 require_relative "../test_retries/component"
 require_relative "../test_retries/null_component"
 require_relative "../test_visibility/component"
@@ -20,12 +22,13 @@ require_relative "../test_visibility/known_tests"
 require_relative "../test_visibility/null_component"
 require_relative "../test_visibility/serializers/factories/test_level"
 require_relative "../test_visibility/serializers/factories/test_suite_level"
+require_relative "../test_visibility/null_transport"
 require_relative "../test_visibility/transport"
 require_relative "../transport/adapters/telemetry_webmock_safe_adapter"
-require_relative "../test_visibility/null_transport"
 require_relative "../transport/api/builder"
 require_relative "../utils/parsing"
 require_relative "../utils/test_run"
+require_relative "../async_writer"
 require_relative "../worker"
 
 module Datadog
@@ -33,7 +36,8 @@ module Datadog
     module Configuration
       # Adds CI behavior to Datadog trace components
       module Components
-        attr_reader :test_visibility, :test_optimisation, :git_tree_upload_worker, :ci_remote, :test_retries, :test_management
+        attr_reader :test_visibility, :test_optimisation, :git_tree_upload_worker, :ci_remote, :test_retries,
+          :test_management, :agentless_logs_submission, :impacted_tests_detection
 
         def initialize(settings)
           @test_optimisation = nil
@@ -42,6 +46,7 @@ module Datadog
           @ci_remote = nil
           @test_retries = TestRetries::NullComponent.new
           @test_management = TestManagement::NullComponent.new
+          @impacted_tests_detection = nil
 
           # Activate CI mode if enabled
           if settings.ci.enabled
@@ -56,6 +61,7 @@ module Datadog
 
           @test_visibility&.shutdown!
           @test_optimisation&.shutdown!
+          @agentless_logs_submission&.shutdown!
           @git_tree_upload_worker&.stop
         end
 
@@ -132,6 +138,10 @@ module Datadog
             known_tests_client: build_known_tests_client(settings, test_visibility_api),
             context_service_uri: settings.ci.test_visibility_drb_server_uri
           )
+
+          @agentless_logs_submission = build_agentless_logs_component(settings, test_visibility_api)
+
+          @impacted_tests_detection = ImpactedTestsDetection::Component.new(enabled: settings.ci.impacted_tests_detection_enabled)
         end
 
         def build_test_optimisation(settings, test_visibility_api)
@@ -224,14 +234,14 @@ module Datadog
           # nil means that coverage event will be ignored
           return nil if api.nil? || settings.ci.discard_traces
 
-          TestOptimisation::Coverage::Writer.new(
+          AsyncWriter.new(
             transport: TestOptimisation::Coverage::Transport.new(api: api)
           )
         end
 
         def build_git_upload_worker(settings, api)
           if settings.ci.git_metadata_upload_enabled
-            git_tree_uploader = Git::TreeUploader.new(api: api)
+            git_tree_uploader = Git::TreeUploader.new(api: api, force_unshallow: settings.ci.impacted_tests_detection_enabled)
             Worker.new do |repository_url|
               git_tree_uploader.call(repository_url)
             end
@@ -256,6 +266,29 @@ module Datadog
           )
         end
 
+        def build_agentless_logs_component(settings, api)
+          if settings.ci.agentless_logs_submission_enabled && !settings.ci.agentless_mode_enabled
+            Datadog.logger.warn(
+              "Agentless logs submission is enabled but agentless mode is not enabled. " \
+              "Logs will not be submitted. " \
+              "Please make sure to set DD_CIVISIBILITY_AGENTLESS_ENABLED to true if you want to submit logs in agentless mode. " \
+              "Otherwise, set DD_AGENTLESS_LOG_SUBMISSION_ENABLED to 0 and use Datadog Agent to submit logs."
+            )
+            settings.ci.agentless_logs_submission_enabled = false
+          end
+
+          Logs::Component.new(
+            enabled: settings.ci.agentless_logs_submission_enabled,
+            writer: build_logs_writer(settings, api)
+          )
+        end
+
+        def build_logs_writer(settings, api)
+          return nil if api.nil? || settings.ci.discard_traces
+
+          AsyncWriter.new(transport: Logs::Transport.new(api: api), options: {buffer_size: 1024})
+        end
+
         # fetch custom tags provided by the user in DD_TAGS env var
         # with prefix test.configuration.
         def custom_configuration(settings)
@@ -294,13 +327,21 @@ module Datadog
           settings.telemetry.shutdown_timeout_seconds = 60.0
 
           begin
-            require "datadog/core/telemetry/http/adapters/net"
+            require "datadog/core/transport/http/adapters/net"
 
-            # patch gem's telemetry transport layer to use Net::HTTP instead of WebMock's Net::HTTP
-            Core::Telemetry::Http::Adapters::Net.include(CI::Transport::Adapters::TelemetryWebmockSafeAdapter)
-          rescue => e
+            # patch gem's core transport layer to use Net::HTTP instead of WebMock's Net::HTTP
+            Core::Transport::HTTP::Adapters::Net.include(CI::Transport::Adapters::TelemetryWebmockSafeAdapter)
+          rescue LoadError, StandardError => e
             Datadog.logger.warn("Failed to patch Datadog gem's telemetry layer: #{e}")
           end
+
+          # for compatibility with old telemetry transport
+          begin
+            require "datadog/core/telemetry/http/adapters/net"
+            Core::Telemetry::Http::Adapters::Net.include(CI::Transport::Adapters::TelemetryWebmockSafeAdapter)
+          rescue LoadError, StandardError => e
+            Datadog.logger.debug("The old telemetry transport layer is not available: #{e}")
+          end
         end
 
         # When timecop is present:

data/lib/datadog/ci/configuration/settings.rb

@@ -128,6 +128,17 @@ module Datadog
                 o.default 20
               end
 
+              option :agentless_logs_submission_enabled do |o|
+                o.type :bool
+                o.env CI::Ext::Settings::ENV_AGENTLESS_LOGS_SUBMISSION_ENABLED
+                o.default false
+              end
+
+              option :agentless_logs_submission_url do |o|
+                o.type :string, nilable: true
+                o.env CI::Ext::Settings::ENV_AGENTLESS_LOGS_SUBMISSION_URL
+              end
+
               # internal only
               option :discard_traces do |o|
                 o.type :bool
@@ -140,6 +151,12 @@ module Datadog
                 o.env CI::Ext::Settings::ENV_TEST_VISIBILITY_DRB_SERVER_URI
               end
 
+              option :impacted_tests_detection_enabled do |o|
+                o.type :bool
+                o.env CI::Ext::Settings::ENV_IMPACTED_TESTS_DETECTION_ENABLED
+                o.default false
+              end
+
               define_method(:instrument) do |integration_name, options = {}, &block|
                 return unless enabled
 

data/lib/datadog/ci/contrib/activesupport/configuration/settings.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "../ext"
+require_relative "../../settings"
+require_relative "../../../utils/configuration"
+
+module Datadog
+  module CI
+    module Contrib
+      module ActiveSupport
+        module Configuration
+          # Custom settings for the ActiveSupport integration
+          # @public_api
+          class Settings < Datadog::CI::Contrib::Settings
+            option :enabled do |o|
+              o.type :bool
+              o.env Ext::ENV_ENABLED
+              o.default true
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/activesupport/ext.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module ActiveSupport
+        # Datadog ActiveSupport integration constants
+        module Ext
+          ENV_ENABLED = "DD_CI_ACTIVESUPPORT_ENABLED"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/activesupport/integration.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "../integration"
+require_relative "configuration/settings"
+require_relative "patcher"
+
+module Datadog
+  module CI
+    module Contrib
+      module ActiveSupport
+        # Description of ActiveSupport integration
+        class Integration < Datadog::CI::Contrib::Integration
+          MINIMUM_VERSION = Gem::Version.new("5.0")
+
+          def version
+            Gem.loaded_specs["activesupport"]&.version
+          end
+
+          def loaded?
+            !defined?(::Rails).nil? && !defined?(::ActiveSupport).nil? && !defined?(::ActiveSupport::TaggedLogging).nil? &&
+              !defined?(::ActiveSupport::TaggedLogging::Formatter).nil?
+          end
+
+          def compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          def late_instrument?
+            true
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/activesupport/logs_formatter.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module ActiveSupport
+        module LogsFormatter
+          def call(severity, timestamp, progname, msg)
+            # don't even construct an object for every log message if agentless logs submission is not enabled
+            return super unless datadog_logs_component.enabled
+            return super unless datadog_configuration[:enabled]
+
+            # additional precaution because we cannot use targeted prepend in Ruby 2.7, so method :tags_text might
+            # not be available (highly unlikely, but not unimaginable)
+            #
+            # (see Datadog::CI::Contrib::ActiveSupport::Patcher for explanation)
+            return super unless respond_to?(:tags_text)
+
+            message = "#{msg} #{tags_text}"
+            return super unless message.include?("dd.trace_id")
+
+            datadog_logs_component.write({
+              message: message,
+              level: severity
+            })
+
+            super
+          end
+
+          def datadog_logs_component
+            Datadog.send(:components).agentless_logs_submission
+          end
+
+          def datadog_configuration
+            Datadog.configuration.ci[:activesupport]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/activesupport/patcher.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative "../patcher"
+require_relative "logs_formatter"
+
+module Datadog
+  module CI
+    module Contrib
+      module ActiveSupport
+        # Patcher enables patching of activesupport module
+        module Patcher
+          include Datadog::CI::Contrib::Patcher
+
+          module_function
+
+          def patch
+            unless datadog_logs_component.enabled
+              Datadog.logger.debug("Datadog logs submission is disabled, skipping activesupport patching")
+              return
+            end
+
+            unless ::Rails.logger.formatter.is_a?(::ActiveSupport::TaggedLogging::Formatter)
+              Datadog.logger.debug {
+                "Rails logger formatter is not an instance of ActiveSupport::TaggedLogging::Formatter, skipping activesupport patching. " \
+                "Formatter: #{::Rails.logger.formatter.class}"
+              }
+              return
+            end
+
+            # Before Ruby 3.0, prepending to a module did not change existing instances where this module was included
+            # It means that for Ruby 2.7 we have to patch formatter's class directly
+            #
+            # Context:
+            # - https://bugs.ruby-lang.org/issues/9573
+            # - https://rubyreferences.github.io/rubychanges/3.0.html#include-and-prepend-now-affects-modules-including-the-receiver
+            if RUBY_VERSION.start_with?("2.7")
+              Rails.logger.formatter.class.prepend(LogsFormatter)
+            else
+              ::ActiveSupport::TaggedLogging::Formatter.prepend(LogsFormatter)
+            end
+          end
+
+          def datadog_logs_component
+            Datadog.send(:components).agentless_logs_submission
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/knapsack/patcher.rb

@@ -9,9 +9,7 @@ module Datadog
         module Patcher
           include Datadog::CI::Contrib::Patcher
 
-          module_function
-
-          def patch
+          def self.patch
             if ::RSpec::Core::Runner.ancestors.include?(::KnapsackPro::Extensions::RSpecExtension::Runner)
               # knapsack already patched rspec runner
               require_relative "runner"

data/lib/datadog/ci/contrib/knapsack/runner.rb

@@ -19,6 +19,7 @@ module Datadog
               return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled]
               return super unless datadog_configuration[:enabled]
 
+              # @type var test_session: Datadog::CI::TestSession?
               test_session = test_visibility_component.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => CI::Contrib::RSpec::Ext::FRAMEWORK,
@@ -27,6 +28,7 @@ module Datadog
                 service: datadog_configuration[:service_name]
               )
 
+              # @type var test_module: Datadog::CI::TestModule?
               test_module = test_visibility_component.start_test_module(CI::Contrib::RSpec::Ext::FRAMEWORK)
 
               result = super

data/lib/datadog/ci/contrib/lograge/configuration/settings.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "../ext"
+require_relative "../../settings"
+require_relative "../../../utils/configuration"
+
+module Datadog
+  module CI
+    module Contrib
+      module Lograge
+        module Configuration
+          # Custom settings for the Lograge integration
+          # @public_api
+          class Settings < Datadog::CI::Contrib::Settings
+            option :enabled do |o|
+              o.type :bool
+              o.env Ext::ENV_ENABLED
+              o.default true
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/lograge/ext.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module Lograge
+        # Datadog Lograge integration constants
+        module Ext
+          ENV_ENABLED = "DD_CI_LOGRAGE_ENABLED"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/lograge/integration.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "../integration"
+require_relative "configuration/settings"
+require_relative "patcher"
+
+module Datadog
+  module CI
+    module Contrib
+      module Lograge
+        # Description of Lograge integration
+        class Integration < Datadog::CI::Contrib::Integration
+          MINIMUM_VERSION = Gem::Version.new("0.14.0")
+
+          def version
+            Gem.loaded_specs["lograge"]&.version
+          end
+
+          def loaded?
+            !defined?(::Lograge).nil? && !defined?(::Lograge::LogSubscribers).nil? &&
+              !defined?(::Lograge::LogSubscribers::Base).nil?
+          end
+
+          def compatible?
+            super && version && version >= MINIMUM_VERSION
+          end
+
+          def late_instrument?
+            true
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end