datadog-ci 1.17.0 → 1.18.0

data/lib/datadog/ci/contrib/rspec/example.rb

@@ -2,6 +2,7 @@
 
 require_relative "../../ext/test"
 require_relative "../../git/local_repository"
+require_relative "../../utils/source_code"
 require_relative "../../utils/test_run"
 require_relative "../instrumentation"
 require_relative "ext"
@@ -26,17 +27,23 @@ module Datadog
               # don't report test to RSpec::Core::Reporter until retries are done
               @skip_reporting = true
 
+              # @type var tags : Hash[String, String]
+              tags = {
+                CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
+                CI::Ext::Test::TAG_FRAMEWORK_VERSION => datadog_integration.version.to_s,
+                CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(metadata[:file_path]),
+                CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s,
+                CI::Ext::Test::TAG_PARAMETERS => datadog_test_parameters
+              }
+
+              end_line = Utils::SourceCode.last_line(@example_block)
+              tags[CI::Ext::Test::TAG_SOURCE_END] = end_line.to_s if end_line
+
               test_retries_component.with_retries do
                 test_visibility_component.trace_test(
                   datadog_test_name,
                   datadog_test_suite_name,
-                  tags: {
-                    CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                    CI::Ext::Test::TAG_FRAMEWORK_VERSION => datadog_integration.version.to_s,
-                    CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(metadata[:file_path]),
-                    CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s,
-                    CI::Ext::Test::TAG_PARAMETERS => datadog_test_parameters
-                  },
+                  tags: tags,
                   service: datadog_configuration[:service_name]
                 ) do |test_span|
                   test_span&.itr_unskippable! if datadog_unskippable?

data/lib/datadog/ci/ext/telemetry.rb

@@ -68,8 +68,6 @@ module Datadog
 
         METRIC_TEST_SESSION = "test_session"
 
-        METRIC_IMPACTED_TESTS_IS_MODIFIED = "impacted_tests.is_modified"
-
         TAG_TEST_FRAMEWORK = "test_framework"
         TAG_EVENT_TYPE = "event_type"
         TAG_HAS_CODEOWNER = "has_codeowner"
@@ -92,6 +90,7 @@ module Datadog
         TAG_IS_QUARANTINED = "is_quarantined"
         TAG_IS_TEST_DISABLED = "is_disabled"
         TAG_HAS_FAILED_ALL_RETRIES = "has_failed_all_retries"
+        TAG_IS_MODIFIED = "is_modified"
         # tags for git_requests.settings_response metric
         TAG_COVERAGE_ENABLED = "coverage_enabled"
         TAG_ITR_ENABLED = "itr_enabled"

data/lib/datadog/ci/ext/test.rb

@@ -20,6 +20,7 @@ module Datadog
         TAG_COMMAND = "test.command"
         TAG_SOURCE_FILE = "test.source.file"
         TAG_SOURCE_START = "test.source.start"
+        TAG_SOURCE_END = "test.source.end"
         TAG_CODEOWNERS = "test.codeowners"
         TAG_PARAMETERS = "test.parameters"
 

data/lib/datadog/ci/git/base_branch_sha_detection/base.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative "../cli"
+
+module Datadog
+  module CI
+    module Git
+      module BaseBranchShaDetection
+        class Base
+          attr_reader :remote_name
+          attr_reader :source_branch
+
+          def initialize(remote_name, source_branch)
+            @remote_name = remote_name
+            @source_branch = source_branch
+          end
+
+          def call
+            raise NotImplementedError, "Subclasses must implement #call"
+          end
+
+          protected
+
+          def merge_base_sha(branch, source_branch)
+            CLI.exec_git_command(["merge-base", branch, source_branch], timeout: CLI::LONG_TIMEOUT)&.strip
+          rescue CLI::GitCommandExecutionError => e
+            Datadog.logger.debug { "Merge base calculation failed for branches '#{branch}' and '#{source_branch}': #{e}" }
+            nil
+          end
+
+          def check_and_fetch_branch(branch, remote_name)
+            # @type var short_branch_name: String
+            short_branch_name = remove_remote_prefix(branch, remote_name)
+
+            # Check if branch already fetched from remote
+            CLI.exec_git_command(["show-ref", "--verify", "--quiet", "refs/remotes/#{remote_name}/#{short_branch_name}"])
+            Datadog.logger.debug { "Branch '#{remote_name}/#{short_branch_name}' already fetched from remote, skipping" }
+          rescue CLI::GitCommandExecutionError => e
+            Datadog.logger.debug { "Branch '#{remote_name}/#{short_branch_name}' doesn't exist locally, checking remote..." }
+
+            begin
+              remote_heads = CLI.exec_git_command(["ls-remote", "--heads", remote_name, short_branch_name])
+              if remote_heads.nil? || remote_heads.empty?
+                Datadog.logger.debug { "Branch '#{remote_name}/#{short_branch_name}' doesn't exist in remote" }
+                return
+              end
+
+              Datadog.logger.debug { "Branch '#{remote_name}/#{short_branch_name}' exists in remote, fetching" }
+              CLI.exec_git_command(["fetch", "--depth", "1", remote_name, short_branch_name], timeout: CLI::LONG_TIMEOUT)
+            rescue CLI::GitCommandExecutionError => e
+              Datadog.logger.debug { "Branch '#{remote_name}/#{short_branch_name}' couldn't be fetched from remote: #{e}" }
+            end
+          end
+
+          def remove_remote_prefix(branch_name, remote_name)
+            branch_name&.sub(/^#{Regexp.escape(remote_name)}\//, "")
+          end
+
+          def branches_equal?(branch_name, default_branch, remote_name)
+            remove_remote_prefix(branch_name, remote_name) == remove_remote_prefix(default_branch, remote_name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/git/base_branch_sha_detection/branch_metric.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Git
+      module BaseBranchShaDetection
+        # Represents metrics for a git branch comparison
+        # including how far behind/ahead it is from a source branch
+        # and the common base commit SHA
+        class BranchMetric
+          attr_reader :branch_name, :behind, :ahead, :base_sha
+
+          def initialize(branch_name:, behind:, ahead:, base_sha:)
+            @branch_name = branch_name
+            @behind = behind
+            @ahead = ahead
+            @base_sha = base_sha
+          end
+
+          # Checks if the branch is up to date with the source branch
+          def up_to_date?
+            @behind == 0 && @ahead == 0
+          end
+
+          # Used for comparison when finding the best branch
+          # Lower divergence score is better
+          def divergence_score
+            @ahead
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/git/base_branch_sha_detection/guesser.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "branch_metric"
+
+require_relative "../local_repository"
+
+module Datadog
+  module CI
+    module Git
+      module BaseBranchShaDetection
+        class Guesser < Base
+          POSSIBLE_BASE_BRANCHES = %w[main master preprod prod dev development trunk].freeze
+          DEFAULT_LIKE_BRANCH_FILTER = /^(#{POSSIBLE_BASE_BRANCHES.join("|")}|release\/.*|hotfix\/.*)$/.freeze
+
+          def call
+            # Check and fetch base branches if they don't exist in local git repository
+            check_and_fetch_base_branches(POSSIBLE_BASE_BRANCHES, remote_name)
+
+            candidates = build_candidate_list(remote_name)
+
+            if candidates.nil? || candidates.empty?
+              Datadog.logger.debug { "No candidate branches found." }
+              return nil
+            end
+
+            metrics = compute_branch_metrics(candidates, source_branch)
+            Datadog.logger.debug { "Branch metrics: '#{metrics}'" }
+
+            best_branch_sha = find_best_branch(metrics, remote_name)
+            Datadog.logger.debug { "Best branch SHA: '#{best_branch_sha}'" }
+
+            best_branch_sha
+          end
+
+          private
+
+          def check_and_fetch_base_branches(branches, remote_name)
+            branches.each do |branch|
+              check_and_fetch_branch(branch, remote_name)
+            end
+          end
+
+          def main_like_branch?(branch_name, remote_name)
+            short_branch_name = remove_remote_prefix(branch_name, remote_name)
+            short_branch_name&.match?(DEFAULT_LIKE_BRANCH_FILTER)
+          end
+
+          def detect_default_branch(remote_name)
+            # @type var default_branch: String?
+            default_branch = nil
+            begin
+              default_ref = CLI.exec_git_command(["symbolic-ref", "--quiet", "--short", "refs/remotes/#{remote_name}/HEAD"])
+              default_branch = remove_remote_prefix(default_ref, remote_name) unless default_ref.nil?
+            rescue
+              Datadog.logger.debug { "Could not get symbolic-ref, trying to find a fallback (main, master)..." }
+            end
+
+            default_branch = find_fallback_default_branch(remote_name) if default_branch.nil?
+            default_branch
+          end
+
+          def find_fallback_default_branch(remote_name)
+            ["main", "master"].each do |fallback|
+              CLI.exec_git_command(["show-ref", "--verify", "--quiet", "refs/remotes/#{remote_name}/#{fallback}"])
+              Datadog.logger.debug { "Found fallback default branch '#{fallback}'" }
+              return fallback
+            rescue
+              next
+            end
+            nil
+          end
+
+          def build_candidate_list(remote_name)
+            # we cannot assume that local branches are the same as remote branches
+            # so we need to go over remote branches only
+            candidates = CLI.exec_git_command(["for-each-ref", "--format=%(refname:short)", "refs/remotes/#{remote_name}"])&.lines&.map(&:strip)
+            Datadog.logger.debug { "Available branches: '#{candidates}'" }
+            candidates&.select! do |candidate_branch|
+              main_like_branch?(candidate_branch, remote_name)
+            end
+            Datadog.logger.debug { "Candidate branches: '#{candidates}'" }
+            candidates
+          end
+
+          def compute_branch_metrics(candidates, source_branch)
+            metrics = []
+            candidates.each do |cand|
+              base_sha = merge_base_sha(cand, source_branch)
+              next if base_sha.nil? || base_sha.empty?
+
+              rev_list_output = CLI.exec_git_command(["rev-list", "--left-right", "--count", "#{cand}...#{source_branch}"], timeout: CLI::LONG_TIMEOUT)&.strip
+              next if rev_list_output.nil?
+
+              behind, ahead = rev_list_output.split.map(&:to_i)
+              next if behind.nil? || ahead.nil?
+
+              metric = BranchMetric.new(
+                branch_name: cand,
+                behind: behind,
+                ahead: ahead,
+                base_sha: base_sha
+              )
+
+              if metric.up_to_date?
+                Datadog.logger.debug { "Branch '#{cand}' is up to date with '#{source_branch}'" }
+                next
+              end
+
+              metrics << metric
+            end
+            metrics
+          end
+
+          def find_best_branch(metrics, remote_name)
+            return nil if metrics.empty?
+
+            # If there's only one metric, return its base SHA
+            return metrics.first.base_sha if metrics.size == 1
+
+            default_branch = detect_default_branch(remote_name)
+            Datadog.logger.debug { "Default branch: '#{default_branch}'" }
+
+            best_metric = metrics.min_by do |metric|
+              [
+                metric.divergence_score,
+                branches_equal?(metric.branch_name, default_branch, remote_name) ? 0 : 1 # prefer default branch on tie
+              ]
+            end
+
+            best_metric&.base_sha
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/git/base_branch_sha_detection/merge_base_extractor.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../local_repository"
+
+module Datadog
+  module CI
+    module Git
+      module BaseBranchShaDetection
+        class MergeBaseExtractor < Base
+          attr_reader :base_branch
+
+          def initialize(remote_name, source_branch, base_branch)
+            super(remote_name, source_branch)
+
+            @base_branch = base_branch
+          end
+
+          def call
+            check_and_fetch_branch(base_branch, remote_name)
+
+            full_base_branch_name = "#{remote_name}/#{remove_remote_prefix(base_branch, remote_name)}"
+            merge_base_sha(full_base_branch_name, source_branch)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/git/base_branch_sha_detector.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative "base_branch_sha_detection/guesser"
+require_relative "base_branch_sha_detection/merge_base_extractor"
+
+module Datadog
+  module CI
+    module Git
+      module BaseBranchShaDetector
+        def self.base_branch_sha(base_branch)
+          Datadog.logger.debug { "Base branch: '#{base_branch}'" }
+
+          remote_name = get_remote_name
+          Datadog.logger.debug { "Remote name: '#{remote_name}'" }
+
+          source_branch = get_source_branch
+          return nil if source_branch.nil?
+
+          Datadog.logger.debug { "Source branch: '#{source_branch}'" }
+
+          strategy = if base_branch.nil?
+            BaseBranchShaDetection::Guesser.new(remote_name, source_branch)
+          else
+            BaseBranchShaDetection::MergeBaseExtractor.new(remote_name, source_branch, base_branch)
+          end
+
+          strategy.call
+        end
+
+        def self.get_remote_name
+          # Try to find remote from upstream tracking
+          upstream = LocalRepository.get_upstream_branch
+
+          if upstream
+            upstream.split("/").first
+          else
+            # Fallback to first remote if no upstream is set
+            first_remote_value = CLI.exec_git_command(["remote"])&.split("\n")&.first
+            Datadog.logger.debug { "First remote value: '#{first_remote_value}'" }
+            first_remote_value || "origin"
+          end
+        end
+
+        def self.get_source_branch
+          source_branch = CLI.exec_git_command(["rev-parse", "--abbrev-ref", "HEAD"])
+          if source_branch.nil?
+            Datadog.logger.debug { "Could not get current branch" }
+            return nil
+          end
+
+          # Verify the branch exists
+          begin
+            CLI.exec_git_command(["rev-parse", "--verify", "--quiet", source_branch])
+          rescue CLI::GitCommandExecutionError
+            # Branch verification failed
+            return nil
+          end
+          source_branch
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/git/cli.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative "../utils/command"
+
+module Datadog
+  module CI
+    module Git
+      module CLI
+        class GitCommandExecutionError < StandardError
+          attr_reader :output, :command, :status
+          def initialize(message, output:, command:, status:)
+            super(message)
+
+            @output = output
+            @command = command
+            @status = status
+          end
+        end
+
+        # Timeout constants for git commands (in seconds)
+        # These values were set based on internal telemetry
+        UNSHALLOW_TIMEOUT = 500
+        LONG_TIMEOUT = 30
+        SHORT_TIMEOUT = 3
+
+        # Execute a git command with optional stdin input and timeout
+        #
+        # @param cmd [Array<String>] The git command as an array of strings
+        # @param stdin [String, nil] Optional stdin data to pass to the command
+        # @param timeout [Integer] Timeout in seconds for the command execution
+        # @return [String, nil] The command output, or nil if the output is empty
+        # @raise [GitCommandExecutionError] If the command fails or times out
+        def self.exec_git_command(cmd, stdin: nil, timeout: SHORT_TIMEOUT)
+          # @type var out: String
+          # @type var status: Process::Status?
+          out, status = Utils::Command.exec_command(["git"] + cmd, stdin_data: stdin, timeout: timeout)
+
+          if status.nil? || !status.success?
+            # Convert command to string representation for error message
+            cmd_str = cmd.join(" ")
+            raise GitCommandExecutionError.new(
+              "Failed to run git command [#{cmd_str}] with input [#{stdin}] and output [#{out}]. Status: #{status}",
+              output: out,
+              command: cmd_str,
+              status: status
+            )
+          end
+
+          return nil if out.empty?
+
+          out
+        end
+      end
+    end
+  end
+end