datadog-ci 1.17.0 → 1.19.0

data/lib/datadog/ci/impacted_tests_detection/component.rb

@@ -1,10 +1,7 @@
 # frozen_string_literal: true
 
-require "set"
-
 require_relative "../ext/test"
 require_relative "../git/local_repository"
-require_relative "telemetry"
 
 module Datadog
   module CI
@@ -12,7 +9,7 @@ module Datadog
       class Component
         def initialize(enabled:)
           @enabled = enabled
-          @changed_files = Set.new
+          @git_diff = Git::Diff.new
         end
 
         def configure(library_settings, test_session)
@@ -30,21 +27,21 @@ module Datadog
             return
           end
 
-          changed_files = Git::LocalRepository.get_changed_files_from_diff(base_commit_sha)
-          if changed_files.nil?
+          git_diff = Git::LocalRepository.get_changes_since(base_commit_sha)
+          if git_diff.empty?
             Datadog.logger.debug { "Impacted tests detection disabled: could not get changed files" }
             @enabled = false
             return
           end
 
           Datadog.logger.debug do
-            "Impacted tests detection: found #{changed_files.size} changed files"
+            "Impacted tests detection: found #{git_diff.size} changed files"
           end
           Datadog.logger.debug do
-            "Impacted tests detection: changed files: #{changed_files.inspect}"
+            "Impacted tests detection: changed files: #{git_diff.inspect}"
           end
 
-          @changed_files = changed_files
+          @git_diff = git_diff
           @enabled = true
         end
 
@@ -58,7 +55,15 @@ module Datadog
           source_file = test_span.source_file
           return false if source_file.nil?
 
-          @changed_files.include?(source_file)
+          # convert to relative path without leading slash
+          # @type var source_file: String
+          source_file = source_file[1..] if source_file.start_with?("/")
+
+          result = @git_diff.lines_changed?(source_file, start_line: test_span.start_line, end_line: test_span.end_line)
+          Datadog.logger.debug do
+            "Impacted tests detection: test #{test_span.name} with source file #{source_file} is modified: #{result}"
+          end
+          result
         end
 
         def tag_modified_test(test_span)
@@ -69,7 +74,6 @@ module Datadog
           end
 
           test_span.set_tag(Ext::Test::TAG_TEST_IS_MODIFIED, "true")
-          Telemetry.impacted_test_detected
         end
 
         private

data/lib/datadog/ci/test.rb

@@ -66,6 +66,22 @@ module Datadog
         get_tag(Ext::Test::TAG_TEST_SESSION_ID)
       end
 
+      # Returns the starting line number of the test in the source file.
+      # @return [Integer] the starting line number
+      # @return [nil] if the starting line is not available
+      def start_line
+        line = get_tag(Ext::Test::TAG_SOURCE_START)
+        line&.to_i
+      end
+
+      # Returns the ending line number of the test in the source file.
+      # @return [Integer] the ending line number
+      # @return [nil] if the ending line is not available
+      def end_line
+        line = get_tag(Ext::Test::TAG_SOURCE_END)
+        line&.to_i
+      end
+
       # Returns "true" if test span represents a retry.
       # @return [Boolean] true if this test is a retry, false otherwise.
       def is_retry?

data/lib/datadog/ci/test_optimisation/component.rb

@@ -150,7 +150,7 @@ module Datadog
         def skippable?(datadog_test_id)
           return false if !enabled? || !skipping_tests?
 
-          @skippable_tests.include?(datadog_test_id)
+          @mutex.synchronize { @skippable_tests.include?(datadog_test_id) }
         end
 
         def mark_if_skippable(test)
@@ -200,8 +200,10 @@ module Datadog
         end
 
         def restore_state(state)
-          @correlation_id = state[:correlation_id]
-          @skippable_tests = state[:skippable_tests]
+          @mutex.synchronize do
+            @correlation_id = state[:correlation_id]
+            @skippable_tests = state[:skippable_tests]
+          end
         end
 
         def storage_key
@@ -225,7 +227,7 @@ module Datadog
         end
 
         def load_datadog_cov!
-          require "datadog_cov.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+          require "datadog_ci_native.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
 
           Datadog.logger.debug("Loaded Datadog code coverage collector, using coverage mode: #{code_coverage_mode}")
         rescue LoadError => e
@@ -253,8 +255,10 @@ module Datadog
               .fetch_skippable_tests(test_session)
           @skippable_tests_fetch_error = skippable_response.error_message unless skippable_response.ok?
 
-          @correlation_id = skippable_response.correlation_id
-          @skippable_tests = skippable_response.tests
+          @mutex.synchronize do
+            @correlation_id = skippable_response.correlation_id
+            @skippable_tests = skippable_response.tests
+          end
 
           Datadog.logger.debug { "Fetched skippable tests: \n #{@skippable_tests}" }
           Datadog.logger.debug { "Found #{@skippable_tests.count} skippable tests." }

data/lib/datadog/ci/test_optimisation/coverage/ddcov.rb

@@ -5,7 +5,7 @@ module Datadog
     module TestOptimisation
       module Coverage
         # Placeholder for code coverage collection
-        # Implementation in ext/datadog_cov
+        # Implementation in ext/datadog_ci_native/datadog_cov.c
         class DDCov
         end
       end

data/lib/datadog/ci/test_visibility/telemetry.rb

@@ -75,6 +75,9 @@ module Datadog
           tags[Ext::Telemetry::TAG_IS_TEST_DISABLED] = "true" if span.get_tag(Ext::Test::TAG_IS_TEST_DISABLED)
           tags[Ext::Telemetry::TAG_HAS_FAILED_ALL_RETRIES] = "true" if span.get_tag(Ext::Test::TAG_HAS_FAILED_ALL_RETRIES)
 
+          # impacted tests tags
+          tags[Ext::Telemetry::TAG_IS_MODIFIED] = "true" if span.get_tag(Ext::Test::TAG_TEST_IS_MODIFIED)
+
           tags
         end
 

data/lib/datadog/ci/utils/command.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "open3"
+require "datadog/core/utils/time"
+
+module Datadog
+  module CI
+    module Utils
+      # Provides a way to call external commands with timeout
+      module Command
+        DEFAULT_TIMEOUT = 10 # seconds
+        BUFFER_SIZE = 1024
+
+        OPEN_STDIN_RETRY_COUNT = 3
+
+        # Executes a command with optional timeout and stdin data
+        #
+        # @param command [Array<String>] Command to execute.
+        # @param stdin_data [String, nil] Data to write to stdin
+        # @param timeout [Integer] Maximum execution time in seconds
+        # @return [Array<String, Process::Status?>] Output and exit status
+        #
+        # @example Safe usage with array (recommended)
+        #   Command.exec_command(["git", "log", "-n", "1"])
+        #
+        #
+        def self.exec_command(command, stdin_data: nil, timeout: DEFAULT_TIMEOUT)
+          output = +""
+          exit_value = nil
+          timeout_reached = false
+
+          begin
+            start = Core::Utils::Time.get_time
+
+            _, stderrout, thread = popen_with_stdin(command, stdin_data: stdin_data)
+            pid = thread[:pid]
+
+            # wait for output and read from stdout/stderr
+            while (Core::Utils::Time.get_time - start) < timeout
+              # wait for data to appear in stderrout channel
+              # maximum wait time 100ms
+              Kernel.select([stderrout], [], [], 0.1)
+
+              begin
+                output << stderrout.read_nonblock(1024)
+              rescue IO::WaitReadable
+              rescue EOFError
+                # we're done here, we return from this cycle when we processed the whole output of the command
+                break
+              end
+            end
+
+            if (Core::Utils::Time.get_time - start) > timeout
+              timeout_reached = true
+            end
+
+            if thread.alive?
+              begin
+                Process.kill("TERM", pid)
+              rescue
+                # Process already terminated
+              end
+            end
+
+            thread.join(1)
+            exit_value = thread.value
+          rescue Errno::EPIPE
+            return ["Error writing to stdin", nil]
+          ensure
+            stderrout&.close
+          end
+
+          # we read command's output as binary so now we need to set an appropriate encoding for the result
+          encoding = Encoding.default_external
+
+          # Sometimes Encoding.default_external is somehow set to US-ASCII which breaks
+          # commit messages with UTF-8 characters like emojis
+          # We force output's encoding to be UTF-8 in this case
+          # This is safe to do as UTF-8 is compatible with US-ASCII
+          if Encoding.default_external == Encoding::US_ASCII
+            encoding = Encoding::UTF_8
+          end
+
+          output.force_encoding(encoding)
+          output.strip! # There's always a "\n" at the end of the command output
+
+          if timeout_reached && output.empty?
+            output = "Command timed out after #{timeout} seconds"
+          end
+
+          [output, exit_value]
+        end
+
+        def self.popen_with_stdin(command, stdin_data: nil, retries_left: OPEN_STDIN_RETRY_COUNT)
+          stdin = nil
+          result = Open3.popen2e(*command)
+          stdin = result.first
+
+          # write input to stdin
+          begin
+            stdin.write(stdin_data) if stdin_data
+          rescue Errno::EPIPE => e
+            if retries_left > 0
+              return popen_with_stdin(command, stdin_data: stdin_data, retries_left: retries_left - 1)
+            else
+              raise e
+            end
+          end
+
+          result
+        ensure
+          stdin&.close
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/utils/source_code.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Utils
+      module SourceCode
+        begin
+          require "datadog_ci_native.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+
+          LAST_LINE_AVAILABLE = true
+        rescue LoadError
+          LAST_LINE_AVAILABLE = false
+        end
+
+        def self.last_line(target)
+          return nil if target.nil?
+          return nil unless LAST_LINE_AVAILABLE
+
+          # Ruby has outdated RBS for RubyVM::InstructionSequence where method `of` is not defined
+          # steep:ignore:start
+          iseq = RubyVM::InstructionSequence.of(target)
+          return nil unless iseq.is_a?(RubyVM::InstructionSequence)
+          # steep:ignore:end
+
+          # this function is implemented in ext/datadog_ci_native/datadog_source_code.c
+          _native_last_line_from_iseq(iseq)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/version.rb

@@ -4,7 +4,7 @@ module Datadog
   module CI
     module VERSION
       MAJOR = 1
-      MINOR = 17
+      MINOR = 19
       PATCH = 0
       PRE = nil
       BUILD = nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datadog-ci
 version: !ruby/object:Gem::Version
-  version: 1.17.0
+  version: 1.19.0
 platform: ruby
 authors:
 - Datadog, Inc.
@@ -60,7 +60,7 @@ email:
 executables:
 - ddcirb
 extensions:
-- ext/datadog_cov/extconf.rb
+- ext/datadog_ci_native/extconf.rb
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
@@ -71,8 +71,12 @@ files:
 - NOTICE
 - README.md
 - exe/ddcirb
-- ext/datadog_cov/datadog_cov.c
-- ext/datadog_cov/extconf.rb
+- ext/datadog_ci_native/ci.c
+- ext/datadog_ci_native/datadog_cov.c
+- ext/datadog_ci_native/datadog_cov.h
+- ext/datadog_ci_native/datadog_source_code.c
+- ext/datadog_ci_native/datadog_source_code.h
+- ext/datadog_ci_native/extconf.rb
 - lib/datadog/ci.rb
 - lib/datadog/ci/async_writer.rb
 - lib/datadog/ci/auto_instrument.rb
@@ -187,6 +191,14 @@ files:
 - lib/datadog/ci/ext/telemetry.rb
 - lib/datadog/ci/ext/test.rb
 - lib/datadog/ci/ext/transport.rb
+- lib/datadog/ci/git/base_branch_sha_detection/base.rb
+- lib/datadog/ci/git/base_branch_sha_detection/branch_metric.rb
+- lib/datadog/ci/git/base_branch_sha_detection/guesser.rb
+- lib/datadog/ci/git/base_branch_sha_detection/merge_base_extractor.rb
+- lib/datadog/ci/git/base_branch_sha_detector.rb
+- lib/datadog/ci/git/changed_lines.rb
+- lib/datadog/ci/git/cli.rb
+- lib/datadog/ci/git/diff.rb
 - lib/datadog/ci/git/local_repository.rb
 - lib/datadog/ci/git/packfiles.rb
 - lib/datadog/ci/git/search_commits.rb
@@ -195,7 +207,6 @@ files:
 - lib/datadog/ci/git/upload_packfile.rb
 - lib/datadog/ci/git/user.rb
 - lib/datadog/ci/impacted_tests_detection/component.rb
-- lib/datadog/ci/impacted_tests_detection/telemetry.rb
 - lib/datadog/ci/logs/component.rb
 - lib/datadog/ci/logs/transport.rb
 - lib/datadog/ci/readonly_test_module.rb
@@ -265,11 +276,13 @@ files:
 - lib/datadog/ci/transport/http.rb
 - lib/datadog/ci/transport/telemetry.rb
 - lib/datadog/ci/utils/bundle.rb
+- lib/datadog/ci/utils/command.rb
 - lib/datadog/ci/utils/configuration.rb
 - lib/datadog/ci/utils/file_storage.rb
 - lib/datadog/ci/utils/git.rb
 - lib/datadog/ci/utils/parsing.rb
 - lib/datadog/ci/utils/rum.rb
+- lib/datadog/ci/utils/source_code.rb
 - lib/datadog/ci/utils/stateful.rb
 - lib/datadog/ci/utils/telemetry.rb
 - lib/datadog/ci/utils/test_run.rb

data/lib/datadog/ci/impacted_tests_detection/telemetry.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../ext/telemetry"
-require_relative "../utils/telemetry"
-
-module Datadog
-  module CI
-    module ImpactedTestsDetection
-      module Telemetry
-        def self.impacted_test_detected
-          Utils::Telemetry.inc(Ext::Telemetry::METRIC_IMPACTED_TESTS_IS_MODIFIED, 1)
-        end
-      end
-    end
-  end
-end