datadog-ci 1.15.0 → 1.17.0

data/lib/datadog/ci/git/local_repository.rb

@@ -2,6 +2,7 @@
 
 require "open3"
 require "pathname"
+require "set"
 
 require_relative "../ext/telemetry"
 require_relative "telemetry"
@@ -23,6 +24,8 @@ module Datadog
         end
 
         COMMAND_RETRY_COUNT = 3
+        POSSIBLE_BASE_BRANCHES = %w[main master preprod prod dev development trunk].freeze
+        DEFAULT_LIKE_BRANCH_FILTER = /^(#{POSSIBLE_BASE_BRANCHES.join("|")}|release\/.*|hotfix\/.*)$/.freeze
 
         def self.root
           return @root if defined?(@root)
@@ -63,7 +66,7 @@ module Datadog
             res = pathname.relative_path_from(root_path).to_s
 
             unless defined?(@prefix_to_root)
-              @prefix_to_root = res&.gsub(path, "") if res.end_with?(path)
+              @prefix_to_root = res.gsub(path, "") if res.end_with?(path)
             end
           end
 
@@ -90,6 +93,7 @@ module Datadog
 
         def self.git_repository_url
           Telemetry.git_command(Ext::Telemetry::Command::GET_REPOSITORY)
+          # @type var res: String?
           res = nil
 
           duration_ms = Core::Utils::Time.measure(:float_millisecond) do
@@ -120,6 +124,7 @@ module Datadog
 
         def self.git_branch
           Telemetry.git_command(Ext::Telemetry::Command::GET_BRANCH)
+          # @type var res: String?
           res = nil
 
           duration_ms = Core::Utils::Time.measure(:float_millisecond) do
@@ -177,7 +182,9 @@ module Datadog
         def self.git_commits
           Telemetry.git_command(Ext::Telemetry::Command::GET_LOCAL_COMMITS)
 
+          # @type var output: String?
           output = nil
+
           duration_ms = Core::Utils::Time.measure(:float_millisecond) do
             output = exec_git_command("git log --format=%H -n 1000 --since=\"1 month ago\"")
           end
@@ -186,7 +193,6 @@ module Datadog
 
           return [] if output.nil?
 
-          # @type var output: String
           output.split("\n")
         rescue => e
           log_failure(e, "git commits")
@@ -199,6 +205,7 @@ module Datadog
           included_commits = filter_invalid_commits(included_commits).join(" ")
           excluded_commits = filter_invalid_commits(excluded_commits).map! { |sha| "^#{sha}" }.join(" ")
 
+          # @type var res: String?
           res = nil
 
           duration_ms = Core::Utils::Time.measure(:float_millisecond) do
@@ -264,6 +271,7 @@ module Datadog
 
         def self.git_unshallow
           Telemetry.git_command(Ext::Telemetry::Command::UNSHALLOW)
+          # @type var res: String?
           res = nil
 
           unshallow_command =
@@ -296,7 +304,7 @@ module Datadog
                   nil
                 end
 
-              break unless unshallowing_errored
+              break [] unless unshallowing_errored
             end
           end
 
@@ -304,6 +312,229 @@ module Datadog
           res
         end
 
+        # Returns a Set of normalized file paths changed since the given base_commit.
+        # If base_commit is nil, returns nil. On error, returns nil.
+        def self.get_changed_files_from_diff(base_commit)
+          return nil if base_commit.nil?
+
+          Datadog.logger.debug { "calculating git diff from base_commit: #{base_commit}" }
+
+          Telemetry.git_command(Ext::Telemetry::Command::DIFF)
+
+          begin
+            # 1. Run the git diff command
+
+            # @type var output: String?
+            output = nil
+            duration_ms = Core::Utils::Time.measure(:float_millisecond) do
+              output = exec_git_command("git diff -U0 --word-diff=porcelain #{base_commit} HEAD")
+            end
+            Telemetry.git_command_ms(Ext::Telemetry::Command::DIFF, duration_ms)
+
+            Datadog.logger.debug { "git diff output: #{output}" }
+
+            return nil if output.nil?
+
+            # 2. Parse the output to extract which files changed
+            changed_files = Set.new
+            output.each_line do |line|
+              # Match lines like: diff --git a/foo/bar.rb b/foo/bar.rb
+              # This captures git changes on file level
+              match = /^diff --git a\/(?<file>.+) b\/(?<file2>.+)$/.match(line)
+              if match && match[:file]
+                changed_file = match[:file]
+                # Normalize to repo root
+                normalized_changed_file = relative_to_root(changed_file)
+                changed_files << normalized_changed_file unless normalized_changed_file.nil? || normalized_changed_file.empty?
+
+                Datadog.logger.debug { "matched changed_file: #{changed_file} from line: #{line}" }
+                Datadog.logger.debug { "normalized_changed_file: #{normalized_changed_file}" }
+              end
+            end
+            changed_files
+          rescue => e
+            telemetry_track_error(e, Ext::Telemetry::Command::DIFF)
+            log_failure(e, "get changed files from diff")
+            nil
+          end
+        end
+
+        # On best effort basis determines the git sha of the most likely
+        # base branch for the current PR.
+        def self.base_commit_sha(base_branch: nil)
+          Telemetry.git_command(Ext::Telemetry::Command::BASE_COMMIT_SHA)
+
+          remote_name = get_remote_name
+          Datadog.logger.debug { "Remote name: '#{remote_name}'" }
+
+          source_branch = get_source_branch
+          return nil if source_branch.nil?
+
+          Datadog.logger.debug { "Source branch: '#{source_branch}'" }
+
+          # Early exit if source is a main-like branch
+          if main_like_branch?(source_branch, remote_name)
+            Datadog.logger.debug { "Branch '#{source_branch}' already matches base branch filter (#{DEFAULT_LIKE_BRANCH_FILTER})" }
+            return nil
+          end
+
+          possible_base_branches = base_branch.nil? ? POSSIBLE_BASE_BRANCHES : [base_branch]
+          # Check and fetch base branches if they don't exist in local git repository
+          check_and_fetch_base_branches(possible_base_branches, remote_name)
+
+          default_branch = detect_default_branch(remote_name)
+          Datadog.logger.debug { "Default branch: '#{default_branch}'" }
+
+          candidates = build_candidate_list(remote_name, source_branch, base_branch)
+          if candidates.nil? || candidates.empty?
+            Datadog.logger.debug { "No candidate branches found." }
+            return nil
+          end
+
+          metrics = compute_branch_metrics(candidates, source_branch)
+          Datadog.logger.debug { "Branch metrics: '#{metrics}'" }
+
+          best_branch_sha = find_best_branch(metrics, default_branch, remote_name)
+          Datadog.logger.debug { "Best branch: '#{best_branch_sha}'" }
+
+          best_branch_sha
+        rescue => e
+          telemetry_track_error(e, Ext::Telemetry::Command::BASE_COMMIT_SHA)
+          log_failure(e, "git base ref")
+          nil
+        end
+
+        def self.check_and_fetch_branch(branch, remote_name)
+          # Check if branch exists locally
+          exec_git_command("git show-ref --verify --quiet refs/heads/#{branch}")
+          Datadog.logger.debug { "Branch '#{branch}' exists locally, skipping" }
+        rescue GitCommandExecutionError => e
+          Datadog.logger.debug { "Branch '#{branch}' doesn't exist locally, checking remote: #{e}" }
+          begin
+            remote_heads = exec_git_command("git ls-remote --heads #{remote_name} #{branch}")
+            if remote_heads.nil? || remote_heads.empty?
+              Datadog.logger.debug { "Branch '#{branch}' doesn't exist in remote" }
+              return
+            end
+
+            Datadog.logger.debug { "Branch '#{branch}' exists in remote, fetching" }
+            exec_git_command("git fetch --depth 1 #{remote_name} #{branch}:#{branch}")
+          rescue GitCommandExecutionError => e
+            Datadog.logger.debug { "Branch '#{branch}' couldn't be fetched from remote: #{e}" }
+          end
+        end
+
+        def self.check_and_fetch_base_branches(branches, remote_name)
+          branches.each do |branch|
+            check_and_fetch_branch(branch, remote_name)
+          end
+        end
+
+        def self.get_source_branch
+          source_branch = exec_git_command("git rev-parse --abbrev-ref HEAD")&.strip
+          if source_branch.nil?
+            Datadog.logger.debug { "Could not get current branch" }
+            return nil
+          end
+
+          exec_git_command("git rev-parse --verify --quiet #{source_branch} > /dev/null")
+          source_branch
+        end
+
+        def self.remove_remote_prefix(branch_name, remote_name)
+          branch_name&.sub(/^#{Regexp.escape(remote_name)}\//, "")
+        end
+
+        def self.main_like_branch?(branch_name, remote_name)
+          short_branch_name = remove_remote_prefix(branch_name, remote_name)
+          short_branch_name&.match?(DEFAULT_LIKE_BRANCH_FILTER)
+        end
+
+        def self.detect_default_branch(remote_name)
+          # @type var default_branch: String?
+          default_branch = nil
+          begin
+            default_ref = exec_git_command("git symbolic-ref --quiet --short \"refs/remotes/#{remote_name}/HEAD\" 2>/dev/null")
+            default_branch = remove_remote_prefix(default_ref, remote_name) unless default_ref.nil?
+          rescue
+            Datadog.logger.debug { "Could not get symbolic-ref, trying to find a fallback (main, master)..." }
+          end
+
+          default_branch = find_fallback_default_branch(remote_name) if default_branch.nil?
+          default_branch
+        end
+
+        def self.find_fallback_default_branch(remote_name)
+          ["main", "master"].each do |fallback|
+            exec_git_command("git show-ref --verify --quiet \"refs/remotes/#{remote_name}/#{fallback}\"")
+            Datadog.logger.debug { "Found fallback default branch '#{fallback}'" }
+            return fallback
+          rescue
+            next
+          end
+          nil
+        end
+
+        def self.build_candidate_list(remote_name, source_branch, base_branch)
+          unless base_branch.nil?
+            return [base_branch]
+          end
+
+          candidates = exec_git_command("git for-each-ref --format='%(refname:short)' refs/heads \"refs/remotes/#{remote_name}\"")&.lines&.map(&:strip)
+          Datadog.logger.debug { "Available branches: '#{candidates}'" }
+          candidates&.select! { |b| b.match?(DEFAULT_LIKE_BRANCH_FILTER) && b != source_branch }
+          Datadog.logger.debug { "Candidate branches: '#{candidates}'" }
+          candidates
+        end
+
+        def self.compute_branch_metrics(candidates, source_branch)
+          metrics = {}
+          candidates.each do |cand|
+            base_sha = exec_git_command("git merge-base #{cand} #{source_branch} 2>/dev/null")&.strip
+            next if base_sha.nil? || base_sha.empty?
+
+            behind, ahead = exec_git_command("git rev-list --left-right --count #{cand}...#{source_branch}")&.strip&.split&.map(&:to_i)
+            metrics[cand] = {behind: behind, ahead: ahead, base_sha: base_sha}
+          end
+          metrics
+        end
+
+        def self.find_best_branch(metrics, default_branch, remote_name)
+          return nil if metrics.empty?
+
+          _, best_data = metrics.min_by do |cand, data|
+            [
+              data[:ahead],
+              default_branch?(cand, default_branch, remote_name) ? 0 : 1 # prefer default branch on tie
+            ]
+          end
+
+          best_data ? best_data[:base_sha] : nil
+        end
+
+        def self.default_branch?(branch, default_branch, remote_name)
+          branch == default_branch || branch == "#{remote_name}/#{default_branch}"
+        end
+
+        def self.get_remote_name
+          # Try to find remote from upstream tracking
+          upstream = nil
+          begin
+            upstream = exec_git_command("git rev-parse --abbrev-ref --symbolic-full-name @{upstream}")&.strip
+          rescue => e
+            Datadog.logger.debug { "Error getting upstream: #{e}" }
+          end
+
+          if upstream
+            upstream.split("/").first
+          else
+            # Fallback to first remote if no upstream is set
+            first_remote_value = exec_git_command("git remote")&.split("\n")&.first
+            Datadog.logger.debug { "First remote value: '#{first_remote_value}'" }
+            first_remote_value || "origin"
+          end
+        end
+
         # makes .exec_git_command private to make sure that this method
         # is not called from outside of this module with insecure parameters
         class << self
@@ -316,10 +547,13 @@ module Datadog
           def exec_git_command(cmd, stdin: nil)
             # Shell injection is alleviated by making sure that no outside modules call this method.
             # It is called only internally with static parameters.
-            # no-dd-sa:ruby-security/shell-injection
+
+            # @type var out: String
+            # @type var status: Process::Status?
             out, status = Open3.capture2e(cmd, stdin_data: stdin)
 
             if status.nil?
+              # @type var retry_count: Integer
               retry_count = COMMAND_RETRY_COUNT
               Datadog.logger.debug { "Opening pipe failed, starting retries..." }
               while status.nil? && retry_count.positive?

data/lib/datadog/ci/git/tree_uploader.rb

@@ -15,10 +15,11 @@ module Datadog
   module CI
     module Git
       class TreeUploader
-        attr_reader :api
+        attr_reader :api, :force_unshallow
 
-        def initialize(api:)
+        def initialize(api:, force_unshallow: false)
           @api = api
+          @force_unshallow = force_unshallow
         end
 
         def call(repository_url)
@@ -45,7 +46,13 @@ module Datadog
             # ask the backend for the list of commits it already has
             known_commits, new_commits = fetch_known_commits_and_split(repository_url, latest_commits)
             # if all commits are present in the backend, we don't need to upload anything
-            if new_commits.empty?
+
+            # We optimize unshallowing process by checking the latest available commits with backend:
+            # if they are already known to backend, then we don't have to unshallow.
+            #
+            # Sometimes we need to unshallow anyway: for impacted tests detection feature for example we need
+            # to calculate git diffs locally. In this case we skip the optimization and always unshallow.
+            if new_commits.empty? && !@force_unshallow
               Datadog.logger.debug("No new commits to upload")
               return
             end

data/lib/datadog/ci/impacted_tests_detection/component.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "set"
+
+require_relative "../ext/test"
+require_relative "../git/local_repository"
+require_relative "telemetry"
+
+module Datadog
+  module CI
+    module ImpactedTestsDetection
+      class Component
+        def initialize(enabled:)
+          @enabled = enabled
+          @changed_files = Set.new
+        end
+
+        def configure(library_settings, test_session)
+          @enabled &&= library_settings.impacted_tests_enabled?
+
+          return unless @enabled
+
+          # we must unshallow the repository before trying to find base_commit_sha or executing `git diff` command
+          git_tree_upload_worker.wait_until_done
+
+          base_commit_sha = test_session.base_commit_sha || Git::LocalRepository.base_commit_sha
+          if base_commit_sha.nil?
+            Datadog.logger.debug { "Impacted tests detection disabled: base commit not found" }
+            @enabled = false
+            return
+          end
+
+          changed_files = Git::LocalRepository.get_changed_files_from_diff(base_commit_sha)
+          if changed_files.nil?
+            Datadog.logger.debug { "Impacted tests detection disabled: could not get changed files" }
+            @enabled = false
+            return
+          end
+
+          Datadog.logger.debug do
+            "Impacted tests detection: found #{changed_files.size} changed files"
+          end
+          Datadog.logger.debug do
+            "Impacted tests detection: changed files: #{changed_files.inspect}"
+          end
+
+          @changed_files = changed_files
+          @enabled = true
+        end
+
+        def enabled?
+          @enabled
+        end
+
+        def modified?(test_span)
+          return false unless enabled?
+
+          source_file = test_span.source_file
+          return false if source_file.nil?
+
+          @changed_files.include?(source_file)
+        end
+
+        def tag_modified_test(test_span)
+          return unless modified?(test_span)
+
+          Datadog.logger.debug do
+            "Impacted tests detection: test #{test_span.name} with source file #{test_span.source_file} is modified"
+          end
+
+          test_span.set_tag(Ext::Test::TAG_TEST_IS_MODIFIED, "true")
+          Telemetry.impacted_test_detected
+        end
+
+        private
+
+        def git_tree_upload_worker
+          Datadog.send(:components).git_tree_upload_worker
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/impacted_tests_detection/telemetry.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "../ext/telemetry"
+require_relative "../utils/telemetry"
+
+module Datadog
+  module CI
+    module ImpactedTestsDetection
+      module Telemetry
+        def self.impacted_test_detected
+          Utils::Telemetry.inc(Ext::Telemetry::METRIC_IMPACTED_TESTS_IS_MODIFIED, 1)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/logs/component.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "datadog/core/environment/platform"
+
+module Datadog
+  module CI
+    module Logs
+      class Component
+        attr_reader :enabled
+
+        def initialize(enabled:, writer:)
+          @enabled = enabled && !writer.nil?
+          @writer = writer
+        end
+
+        def write(event)
+          return unless enabled
+
+          add_common_tags!(event)
+          @writer&.write(event)
+
+          nil
+        end
+
+        def shutdown!
+          @writer&.stop
+        end
+
+        private
+
+        def add_common_tags!(event)
+          test_session = test_visibility.active_test_session
+
+          event[:ddsource] ||= "ruby"
+          event[:ddtags] ||= "datadog.product:citest"
+          event[:service] ||= test_session&.service
+          event[:hostname] ||= Datadog::Core::Environment::Platform.hostname
+        end
+
+        def test_visibility
+          ::Datadog.send(:components).test_visibility
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/logs/transport.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "json"
+
+require "datadog/core/chunker"
+
+module Datadog
+  module CI
+    module Logs
+      class Transport
+        DEFAULT_MAX_PAYLOAD_SIZE = 4.5 * 1024 * 1024
+
+        attr_reader :api,
+          :max_payload_size
+
+        def initialize(api:, max_payload_size: DEFAULT_MAX_PAYLOAD_SIZE)
+          @api = api
+          @max_payload_size = max_payload_size
+        end
+
+        def send_events(events)
+          return [] if events.nil? || events.empty?
+
+          Datadog.logger.debug { "[#{self.class.name}] Sending #{events.count} events..." }
+
+          encoded_events = events.filter_map do |event|
+            encoded_event = event.to_json
+            if event_too_large?(event, encoded_event)
+              next
+            end
+
+            encoded_event
+          end
+
+          responses = []
+          Datadog::Core::Chunker.chunk_by_size(encoded_events, max_payload_size).map do |chunk|
+            encoded_payload = pack_events(chunk)
+            Datadog.logger.debug do
+              "[#{self.class.name}] Send chunk of #{chunk.count} events; payload size #{encoded_payload.size}"
+            end
+
+            response = send_payload(encoded_payload)
+
+            responses << response
+          end
+
+          responses
+        end
+
+        private
+
+        def pack_events(encoded_events)
+          "[#{encoded_events.join(",")}]"
+        end
+
+        def event_too_large?(event, encoded_event)
+          return false unless encoded_event.size > max_payload_size
+
+          # This single event is too large, we can't flush it
+          Datadog.logger.debug(
+            "[#{self.class.name}] Dropping event for logs intake. Payload too large: '#{event.inspect}'"
+          )
+
+          true
+        end
+
+        def send_payload(encoded_payload)
+          @api.logs_intake_request(path: "/v1/input", payload: encoded_payload)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/remote/component.rb

@@ -46,7 +46,8 @@ module Datadog
             Worker.new { test_optimisation.configure(@library_configuration, test_session) },
             Worker.new { test_retries.configure(@library_configuration, test_session) },
             Worker.new { test_visibility.configure(@library_configuration, test_session) },
-            Worker.new { test_management.configure(@library_configuration, test_session) }
+            Worker.new { test_management.configure(@library_configuration, test_session) },
+            Worker.new { impacted_tests_detection.configure(@library_configuration, test_session) }
           ]
 
           # launch configuration workers
@@ -89,6 +90,10 @@ module Datadog
           Datadog.send(:components).test_retries
         end
 
+        def impacted_tests_detection
+          Datadog.send(:components).impacted_tests_detection
+        end
+
         def git_tree_upload_worker
           Datadog.send(:components).git_tree_upload_worker
         end

data/lib/datadog/ci/remote/library_settings.rb

@@ -106,6 +106,14 @@ module Datadog
           )
         end
 
+        def impacted_tests_enabled?
+          return @impacted_tests_enabled if defined?(@impacted_tests_enabled)
+
+          @impacted_tests_enabled = Utils::Parsing.convert_to_bool(
+            payload.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_IMPACTED_TESTS_ENABLED_KEY, false)
+          )
+        end
+
         def slow_test_retries
           return @slow_test_retries if defined?(@slow_test_retries)
 

data/lib/datadog/ci/span.rb

@@ -6,6 +6,7 @@ require "datadog/core/environment/platform"
 
 require_relative "ext/test"
 require_relative "utils/test_run"
+require_relative "ext/git"
 
 module Datadog
   module CI
@@ -164,6 +165,12 @@ module Datadog
         tracer_span.get_tag(Ext::Git::TAG_BRANCH)
       end
 
+      # Returns the base commit SHA for the pull request, if available.
+      # @return [String, nil] the base commit SHA or nil if not set.
+      def base_commit_sha
+        tracer_span.get_tag(Ext::Git::TAG_PULL_REQUEST_BASE_BRANCH_SHA)
+      end
+
       # Returns the OS architecture extracted from the environment.
       # @return [String] OS arch.
       def os_architecture

data/lib/datadog/ci/test.rb

@@ -25,6 +25,10 @@ module Datadog
       # Finishes the current test.
       # @return [void]
       def finish
+        if is_retry? && retry_reason.nil?
+          set_tag(Ext::Test::TAG_RETRY_REASON, Ext::Test::RetryReason::RETRY_EXTERNAL)
+        end
+
         test_visibility.deactivate_test
 
         super
@@ -68,6 +72,12 @@ module Datadog
         get_tag(Ext::Test::TAG_IS_RETRY) == "true"
       end
 
+      # Returns string with a reason why test was retried
+      # @return [String] retry reason
+      def retry_reason
+        get_tag(Ext::Test::TAG_RETRY_REASON)
+      end
+
       # Returns "true" if this span represents a test that wasn't known to Datadog before.
       # @return [Boolean] true if this test is a new one, false otherwise.
       def is_new?
@@ -92,6 +102,12 @@ module Datadog
         get_tag(Ext::Test::TAG_IS_ATTEMPT_TO_FIX) == "true"
       end
 
+      # Returns "true" if this test is marked as modified (e.g., impacted by code changes).
+      # @return [Boolean] true if this test is modified, false otherwise.
+      def modified?
+        get_tag(Ext::Test::TAG_TEST_IS_MODIFIED) == "true"
+      end
+
       # Marks this test as unskippable by the Test Impact Analysis.
       # This must be done before the test execution starts.
       #

data/lib/datadog/ci/test_management/tests_properties.rb

@@ -118,7 +118,8 @@ module Datadog
               "type" => Ext::Transport::DD_API_TEST_MANAGEMENT_TESTS_TYPE,
               "attributes" => {
                 "repository_url" => test_session.git_repository_url,
-                "commit_message" => test_session.git_commit_message
+                "commit_message" => test_session.git_commit_message,
+                "sha" => test_session.git_commit_sha
               }
             }
           }.to_json