datadog-ci 1.16.0 → 1.18.0

data/lib/datadog/ci/git/local_repository.rb

@@ -2,8 +2,12 @@
 
 require "open3"
 require "pathname"
+require "set"
 
 require_relative "../ext/telemetry"
+require_relative "../utils/command"
+require_relative "base_branch_sha_detector"
+require_relative "cli"
 require_relative "telemetry"
 require_relative "user"
 
@@ -11,19 +15,6 @@ module Datadog
   module CI
     module Git
       module LocalRepository
-        class GitCommandExecutionError < StandardError
-          attr_reader :output, :command, :status
-          def initialize(message, output:, command:, status:)
-            super(message)
-
-            @output = output
-            @command = command
-            @status = status
-          end
-        end
-
-        COMMAND_RETRY_COUNT = 3
-
         def self.root
           return @root if defined?(@root)
 
@@ -31,7 +22,7 @@ module Datadog
         end
 
         # ATTENTION: this function is running in a hot path
-        # and should be optimized for performance
+        # and must be optimized for performance
         def self.relative_to_root(path)
           return "" if path.nil?
 
@@ -63,7 +54,7 @@ module Datadog
             res = pathname.relative_path_from(root_path).to_s
 
             unless defined?(@prefix_to_root)
-              @prefix_to_root = res&.gsub(path, "") if res.end_with?(path)
+              @prefix_to_root = res.gsub(path, "") if res.end_with?(path)
             end
           end
 
@@ -90,29 +81,30 @@ module Datadog
 
         def self.git_repository_url
           Telemetry.git_command(Ext::Telemetry::Command::GET_REPOSITORY)
+          # @type var res: String?
           res = nil
 
           duration_ms = Core::Utils::Time.measure(:float_millisecond) do
-            res = exec_git_command("git ls-remote --get-url")
+            res = CLI.exec_git_command(["ls-remote", "--get-url"])
           end
 
           Telemetry.git_command_ms(Ext::Telemetry::Command::GET_REPOSITORY, duration_ms)
           res
         rescue => e
           log_failure(e, "git repository url")
-          telemetry_track_error(e, Ext::Telemetry::Command::GET_REPOSITORY)
+          Telemetry.track_error(e, Ext::Telemetry::Command::GET_REPOSITORY)
           nil
         end
 
         def self.git_root
-          exec_git_command("git rev-parse --show-toplevel")
+          CLI.exec_git_command(["rev-parse", "--show-toplevel"])
         rescue => e
           log_failure(e, "git root path")
           nil
         end
 
         def self.git_commit_sha
-          exec_git_command("git rev-parse HEAD")
+          CLI.exec_git_command(["rev-parse", "HEAD"])
         rescue => e
           log_failure(e, "git commit sha")
           nil
@@ -120,29 +112,30 @@ module Datadog
 
         def self.git_branch
           Telemetry.git_command(Ext::Telemetry::Command::GET_BRANCH)
+          # @type var res: String?
           res = nil
 
           duration_ms = Core::Utils::Time.measure(:float_millisecond) do
-            res = exec_git_command("git rev-parse --abbrev-ref HEAD")
+            res = CLI.exec_git_command(["rev-parse", "--abbrev-ref", "HEAD"])
           end
 
           Telemetry.git_command_ms(Ext::Telemetry::Command::GET_BRANCH, duration_ms)
           res
         rescue => e
           log_failure(e, "git branch")
-          telemetry_track_error(e, Ext::Telemetry::Command::GET_BRANCH)
+          Telemetry.track_error(e, Ext::Telemetry::Command::GET_BRANCH)
           nil
         end
 
         def self.git_tag
-          exec_git_command("git tag --points-at HEAD")
+          CLI.exec_git_command(["tag", "--points-at", "HEAD"])
         rescue => e
           log_failure(e, "git tag")
           nil
         end
 
         def self.git_commit_message
-          exec_git_command("git log -n 1 --format=%B")
+          CLI.exec_git_command(["log", "-n", "1", "--format=%B"])
         rescue => e
           log_failure(e, "git commit message")
           nil
@@ -150,7 +143,7 @@ module Datadog
 
         def self.git_commit_users
           # Get committer and author information in one command.
-          output = exec_git_command("git show -s --format='%an\t%ae\t%at\t%cn\t%ce\t%ct'")
+          output = CLI.exec_git_command(["show", "-s", "--format=%an\t%ae\t%at\t%cn\t%ce\t%ct"])
           unless output
             Datadog.logger.debug(
               "Unable to read git commit users: git command output is nil"
@@ -177,39 +170,42 @@ module Datadog
         def self.git_commits
           Telemetry.git_command(Ext::Telemetry::Command::GET_LOCAL_COMMITS)
 
+          # @type var output: String?
           output = nil
+
           duration_ms = Core::Utils::Time.measure(:float_millisecond) do
-            output = exec_git_command("git log --format=%H -n 1000 --since=\"1 month ago\"")
+            output = CLI.exec_git_command(["log", "--format=%H", "-n", "1000", "--since=\"1 month ago\""], timeout: CLI::LONG_TIMEOUT)
           end
 
           Telemetry.git_command_ms(Ext::Telemetry::Command::GET_LOCAL_COMMITS, duration_ms)
 
           return [] if output.nil?
 
-          # @type var output: String
           output.split("\n")
         rescue => e
           log_failure(e, "git commits")
-          telemetry_track_error(e, Ext::Telemetry::Command::GET_LOCAL_COMMITS)
+          Telemetry.track_error(e, Ext::Telemetry::Command::GET_LOCAL_COMMITS)
           []
         end
 
         def self.git_commits_rev_list(included_commits:, excluded_commits:)
           Telemetry.git_command(Ext::Telemetry::Command::GET_OBJECTS)
-          included_commits = filter_invalid_commits(included_commits).join(" ")
-          excluded_commits = filter_invalid_commits(excluded_commits).map! { |sha| "^#{sha}" }.join(" ")
+          included_commits_list = filter_invalid_commits(included_commits)
+          excluded_commits_list = filter_invalid_commits(excluded_commits).map { |sha| "^#{sha}" }
 
+          # @type var res: String?
           res = nil
 
           duration_ms = Core::Utils::Time.measure(:float_millisecond) do
-            res = exec_git_command(
-              "git rev-list " \
-              "--objects " \
-              "--no-object-names " \
-              "--filter=blob:none " \
-              "--since=\"1 month ago\" " \
-              "#{excluded_commits} #{included_commits}"
-            )
+            cmd = [
+              "rev-list",
+              "--objects",
+              "--no-object-names",
+              "--filter=blob:none",
+              "--since=\"1 month ago\""
+            ] + excluded_commits_list + included_commits_list
+
+            res = CLI.exec_git_command(cmd, timeout: CLI::LONG_TIMEOUT)
           end
 
           Telemetry.git_command_ms(Ext::Telemetry::Command::GET_OBJECTS, duration_ms)
@@ -217,7 +213,7 @@ module Datadog
           res
         rescue => e
           log_failure(e, "git commits rev list")
-          telemetry_track_error(e, Ext::Telemetry::Command::GET_OBJECTS)
+          Telemetry.track_error(e, Ext::Telemetry::Command::GET_OBJECTS)
           nil
         end
 
@@ -232,9 +228,10 @@ module Datadog
           Telemetry.git_command(Ext::Telemetry::Command::PACK_OBJECTS)
 
           duration_ms = Core::Utils::Time.measure(:float_millisecond) do
-            exec_git_command(
-              "git pack-objects --compression=9 --max-pack-size=3m #{path}/#{basename}",
-              stdin: commit_tree
+            CLI.exec_git_command(
+              ["pack-objects", "--compression=9", "--max-pack-size=3m", "#{path}/#{basename}"],
+              stdin: commit_tree,
+              timeout: CLI::LONG_TIMEOUT
             )
           end
           Telemetry.git_command_ms(Ext::Telemetry::Command::PACK_OBJECTS, duration_ms)
@@ -242,7 +239,7 @@ module Datadog
           basename
         rescue => e
           log_failure(e, "git generate packfiles")
-          telemetry_track_error(e, Ext::Telemetry::Command::PACK_OBJECTS)
+          Telemetry.track_error(e, Ext::Telemetry::Command::PACK_OBJECTS)
           nil
         end
 
@@ -251,34 +248,31 @@ module Datadog
           res = false
 
           duration_ms = Core::Utils::Time.measure(:float_millisecond) do
-            res = exec_git_command("git rev-parse --is-shallow-repository") == "true"
+            res = CLI.exec_git_command(["rev-parse", "--is-shallow-repository"]) == "true"
           end
           Telemetry.git_command_ms(Ext::Telemetry::Command::CHECK_SHALLOW, duration_ms)
 
           res
         rescue => e
           log_failure(e, "git shallow clone")
-          telemetry_track_error(e, Ext::Telemetry::Command::CHECK_SHALLOW)
+          Telemetry.track_error(e, Ext::Telemetry::Command::CHECK_SHALLOW)
           false
         end
 
         def self.git_unshallow
           Telemetry.git_command(Ext::Telemetry::Command::UNSHALLOW)
+          # @type var res: String?
           res = nil
 
-          unshallow_command =
-            "git fetch " \
-            "--shallow-since=\"1 month ago\" " \
-            "--update-shallow " \
-            "--filter=\"blob:none\" " \
-            "--recurse-submodules=no " \
-            "$(git config --default origin --get clone.defaultRemoteName)"
-
-          unshallow_remotes = [
-            "$(git rev-parse HEAD)",
-            "$(git rev-parse --abbrev-ref --symbolic-full-name @{upstream})",
-            nil
-          ]
+          default_remote = CLI.exec_git_command(["config", "--default", "origin", "--get", "clone.defaultRemoteName"])&.strip
+          head_commit = git_commit_sha
+          upstream_branch = get_upstream_branch
+
+          # Build array of remotes to try, filtering out nil values
+          unshallow_remotes = []
+          unshallow_remotes << head_commit if head_commit
+          unshallow_remotes << upstream_branch if upstream_branch
+          unshallow_remotes << nil # Ensure the loop runs at least once, even if no valid remotes are available. This acts as a fallback mechanism.
 
           duration_ms = Core::Utils::Time.measure(:float_millisecond) do
             unshallow_remotes.each do |remote|
@@ -286,17 +280,27 @@ module Datadog
 
               res =
                 begin
-                  exec_git_command(
-                    "#{unshallow_command} #{remote}"
-                  )
+                  # @type var cmd: Array[String]
+                  cmd = [
+                    "fetch",
+                    "--shallow-since=\"1 month ago\"",
+                    "--update-shallow",
+                    "--filter=blob:none",
+                    "--recurse-submodules=no",
+                    default_remote
+                  ]
+                  cmd << remote if remote
+
+                  CLI.exec_git_command(cmd, timeout: CLI::UNSHALLOW_TIMEOUT)
                 rescue => e
                   log_failure(e, "git unshallow")
-                  telemetry_track_error(e, Ext::Telemetry::Command::UNSHALLOW)
+                  Telemetry.track_error(e, Ext::Telemetry::Command::UNSHALLOW)
                   unshallowing_errored = true
                   nil
                 end
 
-              break unless unshallowing_errored
+              # If the command succeeded, break and return the result
+              break [] if res && !unshallowing_errored
             end
           end
 
@@ -304,71 +308,80 @@ module Datadog
           res
         end
 
-        # makes .exec_git_command private to make sure that this method
-        # is not called from outside of this module with insecure parameters
-        class << self
-          private
+        # Returns a Set of normalized file paths changed since the given base_commit.
+        # If base_commit is nil, returns nil. On error, returns nil.
+        def self.get_changed_files_from_diff(base_commit)
+          return nil if base_commit.nil?
 
-          def filter_invalid_commits(commits)
-            commits.filter { |commit| Utils::Git.valid_commit_sha?(commit) }
-          end
+          Datadog.logger.debug { "calculating git diff from base_commit: #{base_commit}" }
 
-          def exec_git_command(cmd, stdin: nil)
-            # Shell injection is alleviated by making sure that no outside modules call this method.
-            # It is called only internally with static parameters.
-            # no-dd-sa:ruby-security/shell-injection
-            out, status = Open3.capture2e(cmd, stdin_data: stdin)
-
-            if status.nil?
-              retry_count = COMMAND_RETRY_COUNT
-              Datadog.logger.debug { "Opening pipe failed, starting retries..." }
-              while status.nil? && retry_count.positive?
-                # no-dd-sa:ruby-security/shell-injection
-                out, status = Open3.capture2e(cmd, stdin_data: stdin)
-                Datadog.logger.debug { "After retry status is [#{status}]" }
-                retry_count -= 1
-              end
-            end
+          Telemetry.git_command(Ext::Telemetry::Command::DIFF)
 
-            if status.nil? || !status.success?
-              raise GitCommandExecutionError.new(
-                "Failed to run git command [#{cmd}] with input [#{stdin}] and output [#{out}]",
-                output: out,
-                command: cmd,
-                status: status
-              )
-            end
+          begin
+            # 1. Run the git diff command
 
-            # Sometimes Encoding.default_external is somehow set to US-ASCII which breaks
-            # commit messages with UTF-8 characters like emojis
-            # We force output's encoding to be UTF-8 in this case
-            # This is safe to do as UTF-8 is compatible with US-ASCII
-            if Encoding.default_external == Encoding::US_ASCII
-              out = out.force_encoding(Encoding::UTF_8)
+            # @type var output: String?
+            output = nil
+            duration_ms = Core::Utils::Time.measure(:float_millisecond) do
+              output = CLI.exec_git_command(["diff", "-U0", "--word-diff=porcelain", base_commit, "HEAD"], timeout: CLI::LONG_TIMEOUT)
             end
-            out.strip! # There's always a "\n" at the end of the command output
+            Telemetry.git_command_ms(Ext::Telemetry::Command::DIFF, duration_ms)
+
+            Datadog.logger.debug { "git diff output: #{output}" }
+
+            return nil if output.nil?
+
+            # 2. Parse the output to extract which files changed
+            changed_files = Set.new
+            output.each_line do |line|
+              # Match lines like: diff --git a/foo/bar.rb b/foo/bar.rb
+              # This captures git changes on file level
+              match = /^diff --git a\/(?<file>.+?) b\//.match(line)
+              if match && match[:file]
+                changed_file = match[:file]
+                # Normalize to repo root
+                normalized_changed_file = relative_to_root(changed_file)
+                changed_files << normalized_changed_file unless normalized_changed_file.nil? || normalized_changed_file.empty?
+
+                Datadog.logger.debug { "matched changed_file: #{changed_file} from line: #{line}" }
+                Datadog.logger.debug { "normalized_changed_file: #{normalized_changed_file}" }
+              end
+            end
+            changed_files
+          rescue => e
+            Telemetry.track_error(e, Ext::Telemetry::Command::DIFF)
+            log_failure(e, "get changed files from diff")
+            nil
+          end
+        end
 
-            return nil if out.empty?
+        # On best effort basis determines the git sha of the most likely
+        # base branch for the current PR.
+        def self.base_commit_sha(base_branch: nil)
+          Telemetry.git_command(Ext::Telemetry::Command::BASE_COMMIT_SHA)
 
-            out
-          end
+          BaseBranchShaDetector.base_branch_sha(base_branch)
+        rescue => e
+          Telemetry.track_error(e, Ext::Telemetry::Command::BASE_COMMIT_SHA)
+          log_failure(e, "git base ref")
+          nil
+        end
 
-          def log_failure(e, action)
-            Datadog.logger.debug(
-              "Unable to perform #{action}: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
-            )
-          end
+        def self.get_upstream_branch
+          CLI.exec_git_command(["rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{upstream}"])
+        rescue => e
+          Datadog.logger.debug { "Error getting upstream: #{e}" }
+          nil
+        end
 
-          def telemetry_track_error(e, command)
-            case e
-            when Errno::ENOENT
-              Telemetry.git_command_errors(command, executable_missing: true)
-            when GitCommandExecutionError
-              Telemetry.git_command_errors(command, exit_code: e.status&.to_i)
-            else
-              Telemetry.git_command_errors(command, exit_code: -9000)
-            end
-          end
+        def self.filter_invalid_commits(commits)
+          commits.filter { |commit| Utils::Git.valid_commit_sha?(commit) }
+        end
+
+        def self.log_failure(e, action)
+          Datadog.logger.debug(
+            "Unable to perform #{action}: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
+          )
         end
       end
     end

data/lib/datadog/ci/git/telemetry.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require_relative "../utils/telemetry"
+require_relative "cli"
+
 module Datadog
   module CI
     module Git
@@ -21,6 +24,17 @@ module Datadog
           Utils::Telemetry.distribution(Ext::Telemetry::METRIC_GIT_COMMAND_MS, duration_ms, tags_for_command(command))
         end
 
+        def self.track_error(e, command)
+          case e
+          when Errno::ENOENT
+            git_command_errors(command, executable_missing: true)
+          when CLI::GitCommandExecutionError
+            git_command_errors(command, exit_code: e.status&.to_i)
+          else
+            git_command_errors(command, exit_code: -9000)
+          end
+        end
+
         def self.tags_for_command(command)
           {Ext::Telemetry::TAG_COMMAND => command}
         end

data/lib/datadog/ci/git/tree_uploader.rb

@@ -15,10 +15,11 @@ module Datadog
   module CI
     module Git
       class TreeUploader
-        attr_reader :api
+        attr_reader :api, :force_unshallow
 
-        def initialize(api:)
+        def initialize(api:, force_unshallow: false)
           @api = api
+          @force_unshallow = force_unshallow
         end
 
         def call(repository_url)
@@ -45,7 +46,13 @@ module Datadog
             # ask the backend for the list of commits it already has
             known_commits, new_commits = fetch_known_commits_and_split(repository_url, latest_commits)
             # if all commits are present in the backend, we don't need to upload anything
-            if new_commits.empty?
+
+            # We optimize unshallowing process by checking the latest available commits with backend:
+            # if they are already known to backend, then we don't have to unshallow.
+            #
+            # Sometimes we need to unshallow anyway: for impacted tests detection feature for example we need
+            # to calculate git diffs locally. In this case we skip the optimization and always unshallow.
+            if new_commits.empty? && !@force_unshallow
               Datadog.logger.debug("No new commits to upload")
               return
             end

data/lib/datadog/ci/impacted_tests_detection/component.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "set"
+
+require_relative "../ext/test"
+require_relative "../git/local_repository"
+
+module Datadog
+  module CI
+    module ImpactedTestsDetection
+      class Component
+        def initialize(enabled:)
+          @enabled = enabled
+          @changed_files = Set.new
+        end
+
+        def configure(library_settings, test_session)
+          @enabled &&= library_settings.impacted_tests_enabled?
+
+          return unless @enabled
+
+          # we must unshallow the repository before trying to find base_commit_sha or executing `git diff` command
+          git_tree_upload_worker.wait_until_done
+
+          base_commit_sha = test_session.base_commit_sha || Git::LocalRepository.base_commit_sha
+          if base_commit_sha.nil?
+            Datadog.logger.debug { "Impacted tests detection disabled: base commit not found" }
+            @enabled = false
+            return
+          end
+
+          changed_files = Git::LocalRepository.get_changed_files_from_diff(base_commit_sha)
+          if changed_files.nil?
+            Datadog.logger.debug { "Impacted tests detection disabled: could not get changed files" }
+            @enabled = false
+            return
+          end
+
+          Datadog.logger.debug do
+            "Impacted tests detection: found #{changed_files.size} changed files"
+          end
+          Datadog.logger.debug do
+            "Impacted tests detection: changed files: #{changed_files.inspect}"
+          end
+
+          @changed_files = changed_files
+          @enabled = true
+        end
+
+        def enabled?
+          @enabled
+        end
+
+        def modified?(test_span)
+          return false unless enabled?
+
+          source_file = test_span.source_file
+          return false if source_file.nil?
+
+          @changed_files.include?(source_file)
+        end
+
+        def tag_modified_test(test_span)
+          return unless modified?(test_span)
+
+          Datadog.logger.debug do
+            "Impacted tests detection: test #{test_span.name} with source file #{test_span.source_file} is modified"
+          end
+
+          test_span.set_tag(Ext::Test::TAG_TEST_IS_MODIFIED, "true")
+        end
+
+        private
+
+        def git_tree_upload_worker
+          Datadog.send(:components).git_tree_upload_worker
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/remote/component.rb

@@ -46,7 +46,8 @@ module Datadog
             Worker.new { test_optimisation.configure(@library_configuration, test_session) },
             Worker.new { test_retries.configure(@library_configuration, test_session) },
             Worker.new { test_visibility.configure(@library_configuration, test_session) },
-            Worker.new { test_management.configure(@library_configuration, test_session) }
+            Worker.new { test_management.configure(@library_configuration, test_session) },
+            Worker.new { impacted_tests_detection.configure(@library_configuration, test_session) }
           ]
 
           # launch configuration workers
@@ -89,6 +90,10 @@ module Datadog
           Datadog.send(:components).test_retries
         end
 
+        def impacted_tests_detection
+          Datadog.send(:components).impacted_tests_detection
+        end
+
         def git_tree_upload_worker
           Datadog.send(:components).git_tree_upload_worker
         end

data/lib/datadog/ci/remote/library_settings.rb

@@ -106,6 +106,14 @@ module Datadog
           )
         end
 
+        def impacted_tests_enabled?
+          return @impacted_tests_enabled if defined?(@impacted_tests_enabled)
+
+          @impacted_tests_enabled = Utils::Parsing.convert_to_bool(
+            payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_IMPACTED_TESTS_ENABLED_KEY, false)
+          )
+        end
+
         def slow_test_retries
           return @slow_test_retries if defined?(@slow_test_retries)
 

data/lib/datadog/ci/span.rb

@@ -6,6 +6,7 @@ require "datadog/core/environment/platform"
 
 require_relative "ext/test"
 require_relative "utils/test_run"
+require_relative "ext/git"
 
 module Datadog
   module CI
@@ -164,6 +165,12 @@ module Datadog
         tracer_span.get_tag(Ext::Git::TAG_BRANCH)
       end
 
+      # Returns the base commit SHA for the pull request, if available.
+      # @return [String, nil] the base commit SHA or nil if not set.
+      def base_commit_sha
+        tracer_span.get_tag(Ext::Git::TAG_PULL_REQUEST_BASE_BRANCH_SHA)
+      end
+
       # Returns the OS architecture extracted from the environment.
       # @return [String] OS arch.
       def os_architecture

data/lib/datadog/ci/test.rb

@@ -102,6 +102,12 @@ module Datadog
         get_tag(Ext::Test::TAG_IS_ATTEMPT_TO_FIX) == "true"
       end
 
+      # Returns "true" if this test is marked as modified (e.g., impacted by code changes).
+      # @return [Boolean] true if this test is modified, false otherwise.
+      def modified?
+        get_tag(Ext::Test::TAG_TEST_IS_MODIFIED) == "true"
+      end
+
       # Marks this test as unskippable by the Test Impact Analysis.
       # This must be done before the test execution starts.
       #

data/lib/datadog/ci/test_management/tests_properties.rb

@@ -118,7 +118,8 @@ module Datadog
               "type" => Ext::Transport::DD_API_TEST_MANAGEMENT_TESTS_TYPE,
               "attributes" => {
                 "repository_url" => test_session.git_repository_url,
-                "commit_message" => test_session.git_commit_message
+                "commit_message" => test_session.git_commit_message,
+                "sha" => test_session.git_commit_sha
               }
             }
           }.to_json