datadog-ci 1.16.0 → 1.18.0

data/lib/datadog/ci/test_optimisation/component.rb

@@ -150,7 +150,7 @@ module Datadog
         def skippable?(datadog_test_id)
           return false if !enabled? || !skipping_tests?
 
-          @skippable_tests.include?(datadog_test_id)
+          @mutex.synchronize { @skippable_tests.include?(datadog_test_id) }
         end
 
         def mark_if_skippable(test)
@@ -200,8 +200,10 @@ module Datadog
         end
 
         def restore_state(state)
-          @correlation_id = state[:correlation_id]
-          @skippable_tests = state[:skippable_tests]
+          @mutex.synchronize do
+            @correlation_id = state[:correlation_id]
+            @skippable_tests = state[:skippable_tests]
+          end
         end
 
         def storage_key
@@ -225,7 +227,7 @@ module Datadog
         end
 
         def load_datadog_cov!
-          require "datadog_cov.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+          require "datadog_ci_native.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
 
           Datadog.logger.debug("Loaded Datadog code coverage collector, using coverage mode: #{code_coverage_mode}")
         rescue LoadError => e
@@ -253,8 +255,10 @@ module Datadog
               .fetch_skippable_tests(test_session)
           @skippable_tests_fetch_error = skippable_response.error_message unless skippable_response.ok?
 
-          @correlation_id = skippable_response.correlation_id
-          @skippable_tests = skippable_response.tests
+          @mutex.synchronize do
+            @correlation_id = skippable_response.correlation_id
+            @skippable_tests = skippable_response.tests
+          end
 
           Datadog.logger.debug { "Fetched skippable tests: \n #{@skippable_tests}" }
           Datadog.logger.debug { "Found #{@skippable_tests.count} skippable tests." }

data/lib/datadog/ci/test_optimisation/coverage/ddcov.rb

@@ -5,7 +5,7 @@ module Datadog
     module TestOptimisation
       module Coverage
         # Placeholder for code coverage collection
-        # Implementation in ext/datadog_cov
+        # Implementation in ext/datadog_ci_native/datadog_cov.c
         class DDCov
         end
       end

data/lib/datadog/ci/test_retries/component.rb

@@ -2,11 +2,11 @@
 
 require_relative "driver/no_retry"
 require_relative "driver/retry_failed"
-require_relative "driver/retry_new"
+require_relative "driver/retry_flake_detection"
 
 require_relative "strategy/no_retry"
 require_relative "strategy/retry_failed"
-require_relative "strategy/retry_new"
+require_relative "strategy/retry_flake_detection"
 require_relative "strategy/retry_flaky_fixed"
 
 require_relative "../ext/telemetry"
@@ -31,13 +31,13 @@ module Datadog
         )
           no_retries_strategy = Strategy::NoRetry.new
 
-          @retry_failed_strategy = Strategy::RetryFailed.new(
+          retry_failed_strategy = Strategy::RetryFailed.new(
             enabled: retry_failed_tests_enabled,
             max_attempts: retry_failed_tests_max_attempts,
             total_limit: retry_failed_tests_total_limit
           )
 
-          @retry_new_strategy = Strategy::RetryNew.new(
+          retry_flake_detection_strategy = Strategy::RetryFlakeDetection.new(
             enabled: retry_new_tests_enabled
           )
 
@@ -49,8 +49,8 @@ module Datadog
           # order is important, we apply the first matching strategy
           @retry_strategies = [
             retry_flaky_fixed_strategy,
-            @retry_new_strategy,
-            @retry_failed_strategy,
+            retry_flake_detection_strategy,
+            retry_failed_strategy,
             no_retries_strategy
           ]
           @mutex = Mutex.new
@@ -111,23 +111,14 @@ module Datadog
           test_span&.set_tag(Ext::Test::TAG_HAS_FAILED_ALL_RETRIES, "true") if test_span&.all_executions_failed?
 
           # if we are attempting to fix the test and all retries passed, we indicate that the fix might have worked
-          if test_span&.attempt_to_fix? && test_span.all_executions_passed?
-            test_span&.set_tag(Ext::Test::TAG_ATTEMPT_TO_FIX_PASSED, "true")
-          end
+          # otherwise we send "false" to show that it didn't work
+          test_span&.set_tag(Ext::Test::TAG_ATTEMPT_TO_FIX_PASSED, test_span&.all_executions_passed?.to_s) if test_span&.attempt_to_fix?
         end
 
         def should_retry?
           !!current_retry_driver&.should_retry?
         end
 
-        def auto_test_retries_feature_enabled
-          @retry_failed_strategy.enabled
-        end
-
-        def early_flake_detection_feature_enabled
-          @retry_new_strategy.enabled
-        end
-
         private
 
         def current_retry_driver

data/lib/datadog/ci/test_retries/driver/{retry_new.rb → retry_flake_detection.rb}

@@ -9,7 +9,7 @@ module Datadog
     module TestRetries
       module Driver
         # retry every new test up to 10 times (early flake detection)
-        class RetryNew < Base
+        class RetryFlakeDetection < Base
           def initialize(test_span, max_attempts_thresholds:)
             @max_attempts_thresholds = max_attempts_thresholds
             @attempts = 0

data/lib/datadog/ci/test_retries/strategy/{retry_new.rb → retry_flake_detection.rb}

@@ -2,13 +2,13 @@
 
 require_relative "base"
 
-require_relative "../driver/retry_new"
+require_relative "../driver/retry_flake_detection"
 
 module Datadog
   module CI
     module TestRetries
       module Strategy
-        class RetryNew < Base
+        class RetryFlakeDetection < Base
           DEFAULT_TOTAL_TESTS_COUNT = 100
 
           attr_reader :enabled, :max_attempts_thresholds, :total_limit, :retried_count
@@ -31,7 +31,7 @@ module Datadog
               mark_test_session_faulty(Datadog::CI.active_test_session)
             end
 
-            @enabled && !test_span.skipped? && test_span.is_new?
+            @enabled && !test_span.skipped? && (test_span.is_new? || test_span.modified?)
           end
 
           def configure(library_settings, test_session)
@@ -52,7 +52,7 @@ module Datadog
             end
             @retried_count += 1
 
-            Driver::RetryNew.new(test_span, max_attempts_thresholds: @max_attempts_thresholds)
+            Driver::RetryFlakeDetection.new(test_span, max_attempts_thresholds: @max_attempts_thresholds)
           end
 
           private

data/lib/datadog/ci/test_visibility/component.rb

@@ -261,6 +261,8 @@ module Datadog
 
           mark_test_as_new(test) if new_test?(test)
 
+          impacted_tests_detection.tag_modified_test(test)
+
           test_management.tag_test_from_properties(test)
 
           test_optimisation.mark_if_skippable(test)
@@ -435,6 +437,10 @@ module Datadog
           Datadog.send(:components).test_management
         end
 
+        def impacted_tests_detection
+          Datadog.send(:components).impacted_tests_detection
+        end
+
         # DISTRIBUTED RUBY CONTEXT
         def start_drb_service
           return if @context_service_uri

data/lib/datadog/ci/test_visibility/telemetry.rb

@@ -75,6 +75,9 @@ module Datadog
           tags[Ext::Telemetry::TAG_IS_TEST_DISABLED] = "true" if span.get_tag(Ext::Test::TAG_IS_TEST_DISABLED)
           tags[Ext::Telemetry::TAG_HAS_FAILED_ALL_RETRIES] = "true" if span.get_tag(Ext::Test::TAG_HAS_FAILED_ALL_RETRIES)
 
+          # impacted tests tags
+          tags[Ext::Telemetry::TAG_IS_MODIFIED] = "true" if span.get_tag(Ext::Test::TAG_TEST_IS_MODIFIED)
+
           tags
         end
 

data/lib/datadog/ci/utils/command.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require "open3"
+require "datadog/core/utils/time"
+
+module Datadog
+  module CI
+    module Utils
+      # Provides a way to call external commands with timeout
+      module Command
+        DEFAULT_TIMEOUT = 10 # seconds
+        BUFFER_SIZE = 1024
+
+        OPEN_STDIN_RETRY_COUNT = 3
+
+        # Executes a command with optional timeout and stdin data
+        #
+        # @param command [Array<String>] Command to execute.
+        # @param stdin_data [String, nil] Data to write to stdin
+        # @param timeout [Integer] Maximum execution time in seconds
+        # @return [Array<String, Process::Status?>] Output and exit status
+        #
+        # @example Safe usage with array (recommended)
+        #   Command.exec_command(["git", "log", "-n", "1"])
+        #
+        #
+        def self.exec_command(command, stdin_data: nil, timeout: DEFAULT_TIMEOUT)
+          output = +""
+          exit_value = nil
+          timeout_reached = false
+
+          begin
+            start = Core::Utils::Time.get_time
+
+            _, stderrout, thread = popen_with_stdin(command, stdin_data: stdin_data)
+            pid = thread[:pid]
+
+            # wait for output and read from stdout/stderr
+            while (Core::Utils::Time.get_time - start) < timeout
+              # wait for data to appear in stderrout channel
+              # maximum wait time 100ms
+              Kernel.select([stderrout], [], [], 0.1)
+
+              begin
+                output << stderrout.read_nonblock(1024)
+              rescue IO::WaitReadable
+              rescue EOFError
+                # we're done here, we return from this cycle when we processed the whole output of the command
+                break
+              end
+            end
+
+            if (Core::Utils::Time.get_time - start) > timeout
+              timeout_reached = true
+            end
+
+            if thread.alive?
+              begin
+                Process.kill("TERM", pid)
+              rescue
+                # Process already terminated
+              end
+            end
+
+            thread.join(1)
+            exit_value = thread.value
+          rescue Errno::EPIPE
+            return ["Error writing to stdin", nil]
+          ensure
+            stderrout&.close
+          end
+
+          # we read command's output as binary so now we need to set an appropriate encoding for the result
+          encoding = Encoding.default_external
+
+          # Sometimes Encoding.default_external is somehow set to US-ASCII which breaks
+          # commit messages with UTF-8 characters like emojis
+          # We force output's encoding to be UTF-8 in this case
+          # This is safe to do as UTF-8 is compatible with US-ASCII
+          if Encoding.default_external == Encoding::US_ASCII
+            encoding = Encoding::UTF_8
+          end
+
+          output.force_encoding(encoding)
+          output.strip! # There's always a "\n" at the end of the command output
+
+          if timeout_reached && output.empty?
+            output = "Command timed out after #{timeout} seconds"
+          end
+
+          [output, exit_value]
+        end
+
+        def self.popen_with_stdin(command, stdin_data: nil, retries_left: OPEN_STDIN_RETRY_COUNT)
+          result = Open3.popen2e(*command)
+          stdin = result.first
+
+          # write input to stdin
+          begin
+            stdin.write(stdin_data) if stdin_data
+          rescue Errno::EPIPE => e
+            if retries_left > 0
+              return popen_with_stdin(command, stdin_data: stdin_data, retries_left: retries_left - 1)
+            else
+              raise e
+            end
+          end
+
+          result
+        ensure
+          stdin.close
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/utils/source_code.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Utils
+      module SourceCode
+        begin
+          require "datadog_ci_native.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+
+          LAST_LINE_AVAILABLE = true
+        rescue LoadError
+          LAST_LINE_AVAILABLE = false
+        end
+
+        def self.last_line(target)
+          return nil if target.nil?
+          return nil unless LAST_LINE_AVAILABLE
+
+          # Ruby has outdated RBS for RubyVM::InstructionSequence where method `of` is not defined
+          # steep:ignore:start
+          iseq = RubyVM::InstructionSequence.of(target)
+          return nil unless iseq.is_a?(RubyVM::InstructionSequence)
+          # steep:ignore:end
+
+          # this function is implemented in ext/datadog_ci_native/datadog_source_code.c
+          _native_last_line_from_iseq(iseq)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/version.rb

@@ -4,7 +4,7 @@ module Datadog
   module CI
     module VERSION
       MAJOR = 1
-      MINOR = 16
+      MINOR = 18
       PATCH = 0
       PRE = nil
       BUILD = nil

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: datadog-ci
 version: !ruby/object:Gem::Version
-  version: 1.16.0
+  version: 1.18.0
 platform: ruby
 authors:
 - Datadog, Inc.
 bindir: exe
 cert_chain: []
-date: 2025-04-15 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: datadog
@@ -60,7 +60,7 @@ email:
 executables:
 - ddcirb
 extensions:
-- ext/datadog_cov/extconf.rb
+- ext/datadog_ci_native/extconf.rb
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
@@ -71,8 +71,12 @@ files:
 - NOTICE
 - README.md
 - exe/ddcirb
-- ext/datadog_cov/datadog_cov.c
-- ext/datadog_cov/extconf.rb
+- ext/datadog_ci_native/ci.c
+- ext/datadog_ci_native/datadog_cov.c
+- ext/datadog_ci_native/datadog_cov.h
+- ext/datadog_ci_native/datadog_source_code.c
+- ext/datadog_ci_native/datadog_source_code.h
+- ext/datadog_ci_native/extconf.rb
 - lib/datadog/ci.rb
 - lib/datadog/ci/async_writer.rb
 - lib/datadog/ci/auto_instrument.rb
@@ -187,6 +191,12 @@ files:
 - lib/datadog/ci/ext/telemetry.rb
 - lib/datadog/ci/ext/test.rb
 - lib/datadog/ci/ext/transport.rb
+- lib/datadog/ci/git/base_branch_sha_detection/base.rb
+- lib/datadog/ci/git/base_branch_sha_detection/branch_metric.rb
+- lib/datadog/ci/git/base_branch_sha_detection/guesser.rb
+- lib/datadog/ci/git/base_branch_sha_detection/merge_base_extractor.rb
+- lib/datadog/ci/git/base_branch_sha_detector.rb
+- lib/datadog/ci/git/cli.rb
 - lib/datadog/ci/git/local_repository.rb
 - lib/datadog/ci/git/packfiles.rb
 - lib/datadog/ci/git/search_commits.rb
@@ -194,6 +204,7 @@ files:
 - lib/datadog/ci/git/tree_uploader.rb
 - lib/datadog/ci/git/upload_packfile.rb
 - lib/datadog/ci/git/user.rb
+- lib/datadog/ci/impacted_tests_detection/component.rb
 - lib/datadog/ci/logs/component.rb
 - lib/datadog/ci/logs/transport.rb
 - lib/datadog/ci/readonly_test_module.rb
@@ -221,14 +232,14 @@ files:
 - lib/datadog/ci/test_retries/driver/base.rb
 - lib/datadog/ci/test_retries/driver/no_retry.rb
 - lib/datadog/ci/test_retries/driver/retry_failed.rb
+- lib/datadog/ci/test_retries/driver/retry_flake_detection.rb
 - lib/datadog/ci/test_retries/driver/retry_flaky_fixed.rb
-- lib/datadog/ci/test_retries/driver/retry_new.rb
 - lib/datadog/ci/test_retries/null_component.rb
 - lib/datadog/ci/test_retries/strategy/base.rb
 - lib/datadog/ci/test_retries/strategy/no_retry.rb
 - lib/datadog/ci/test_retries/strategy/retry_failed.rb
+- lib/datadog/ci/test_retries/strategy/retry_flake_detection.rb
 - lib/datadog/ci/test_retries/strategy/retry_flaky_fixed.rb
-- lib/datadog/ci/test_retries/strategy/retry_new.rb
 - lib/datadog/ci/test_session.rb
 - lib/datadog/ci/test_suite.rb
 - lib/datadog/ci/test_visibility/component.rb
@@ -263,11 +274,13 @@ files:
 - lib/datadog/ci/transport/http.rb
 - lib/datadog/ci/transport/telemetry.rb
 - lib/datadog/ci/utils/bundle.rb
+- lib/datadog/ci/utils/command.rb
 - lib/datadog/ci/utils/configuration.rb
 - lib/datadog/ci/utils/file_storage.rb
 - lib/datadog/ci/utils/git.rb
 - lib/datadog/ci/utils/parsing.rb
 - lib/datadog/ci/utils/rum.rb
+- lib/datadog/ci/utils/source_code.rb
 - lib/datadog/ci/utils/stateful.rb
 - lib/datadog/ci/utils/telemetry.rb
 - lib/datadog/ci/utils/test_run.rb
@@ -298,7 +311,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 2.0.0
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Datadog Test Optimization for your ruby application
 test_files: []