datadog-ci 1.18.0 → 1.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4673e9680749b01863ddde129169fb985f7be2f46ae83da674b856bc70900dc9
-  data.tar.gz: 5e0c4f0a055d7a3017dc2984d39b142cfcf9f4bfb39dca5713a18af0481276f6
+  metadata.gz: 6c730f78f9fefc725ba4499a38729eff14f5fc48a4a8e6a055ade2b4cedc030a
+  data.tar.gz: 86a9682f7623bc85a8e8a05fd8b1f91ecb12f459e7f2425fd2fb79d4ce217227
 SHA512:
-  metadata.gz: 58d120417a247f9e6be958efeea4094e6d3f5dc861df5dbe74428f7968e2dc96156b19d5c3db512f70afa15686997e6b38aae39dbeb8d0bb2f4f19fdb8b524b4
-  data.tar.gz: 7d91c8fc4e85bede1f57dc64a151d4ebfb222be88710f958e8a6538ad37e44cedf56c6379fb1476fb472ebf65db57023915100254e180e6087b2c7c0eb7d78ca
+  metadata.gz: ebb3fca5a5cf1db85795b017ffefef4bb09560c4c8c65f54b4d4cc88b93016776a861ec5c48bf700bdff4ee18c4416f6aee632b5abd392ae930a1a1b00042f0c
+  data.tar.gz: 9b5b171277dc7bb8510dec4bd332a937431ac4bb3f277c9ad3221dd38f3d5eec27b1c92415aa47c0289259543d2ee21fe8ae1dc532e0f9d9376706a38e59e712

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
 
+## [1.19.0] - 2025-06-16
+
+### Changed
+
+* Impacted tests detection works with line level granularity ([#335][])
+
+### Fixed
+
+* Fix stdin close in Command.popen_with_stdin ([#339][])
+
 ## [1.18.0] - 2025-06-06
 
 ### Added
@@ -453,7 +463,8 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 
 - Ruby versions < 2.7 no longer supported ([#8][])
 
-[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.18.0...main
+[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.19.0...main
+[1.19.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.18.0...v1.19.0
 [1.18.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.17.0...v1.18.0
 [1.17.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.16.0...v1.17.0
 [1.16.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.15.0...v1.16.0
@@ -647,4 +658,6 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 [#321]: https://github.com/DataDog/datadog-ci-rb/issues/321
 [#323]: https://github.com/DataDog/datadog-ci-rb/issues/323
 [#327]: https://github.com/DataDog/datadog-ci-rb/issues/327
-[#329]: https://github.com/DataDog/datadog-ci-rb/issues/329
+[#329]: https://github.com/DataDog/datadog-ci-rb/issues/329
+[#335]: https://github.com/DataDog/datadog-ci-rb/issues/335
+[#339]: https://github.com/DataDog/datadog-ci-rb/issues/339

data/lib/datadog/ci/ext/environment/providers/appveyor.rb

@@ -85,7 +85,7 @@ module Datadog
 
             def git_pull_request_base_branch
               # from docs: build branch. For Pull Request commits it is base branch PR is merging into
-              env["APPVEYOR_REPO_BRANCH"]
+              env["APPVEYOR_REPO_BRANCH"] if env["APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH"]
             end
 
             private

data/lib/datadog/ci/ext/environment/providers/buildkite.rb

@@ -85,6 +85,10 @@ module Datadog
               env["BUILDKITE_MESSAGE"]
             end
 
+            def git_pull_request_base_branch
+              env["BUILDKITE_PULL_REQUEST_BASE_BRANCH"]
+            end
+
             def ci_env_vars
               {
                 "BUILDKITE_BUILD_ID" => env["BUILDKITE_BUILD_ID"],

data/lib/datadog/ci/ext/environment/providers/gitlab.rb

@@ -104,10 +104,6 @@ module Datadog
               env["CI_MERGE_REQUEST_TARGET_BRANCH_NAME"]
             end
 
-            def git_pull_request_base_branch_sha
-              env["CI_MERGE_REQUEST_TARGET_BRANCH_SHA"]
-            end
-
             def git_commit_head_sha
               env["CI_MERGE_REQUEST_SOURCE_BRANCH_SHA"]
             end

data/lib/datadog/ci/git/changed_lines.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Git
+      # Helper class to efficiently store and query changed line intervals for a single file
+      # Uses merged sorted intervals with binary search for O(log n) query performance
+      class ChangedLines
+        def initialize
+          @intervals = [] # Array of [start, end] pairs
+          @built = false
+          @mutex = Mutex.new
+        end
+
+        # Add an interval (defers merging until build! is called)
+        def add_interval(start_line, end_line)
+          return if start_line > end_line
+
+          @mutex.synchronize do
+            @intervals << [start_line, end_line]
+            @built = false
+          end
+        end
+
+        # Sort and merge all intervals
+        # Call this after all intervals have been added
+        def build!
+          @mutex.synchronize do
+            return false if @built
+
+            @built = true
+            return false if @intervals.empty?
+
+            # Sort intervals by start line
+            @intervals.sort_by!(&:first)
+
+            # Merge overlapping intervals
+            merged = []
+
+            # @type var current_start: Integer
+            # @type var current_end: Integer
+            current_start, current_end = @intervals.first
+
+            @intervals.each_with_index do |interval, index|
+              next if index == 0
+              # @type var start_line: Integer
+              # @type var end_line: Integer
+              start_line, end_line = interval
+
+              if start_line <= current_end + 1
+                # Overlapping or adjacent intervals, merge them
+                current_end = [current_end, end_line].max
+              else
+                # Non-overlapping interval, save current and start new
+                merged << [current_start, current_end]
+                current_start = start_line
+                current_end = end_line
+              end
+            end
+
+            merged << [current_start, current_end]
+
+            @intervals = merged
+            true
+          end
+        end
+
+        # Check if any line in the query interval overlaps with changed lines
+        # Uses binary search for O(log n) performance
+        def overlaps?(query_start, query_end)
+          build! unless @built
+
+          return false if @intervals.empty? || query_start > query_end
+
+          # Binary search for the first interval that might overlap
+          left = 0
+          right = @intervals.length - 1
+
+          while left <= right
+            mid = (left + right) / 2
+            # @type var interval_start: Integer
+            # @type var interval_end: Integer
+            interval_start, interval_end = @intervals[mid]
+
+            if interval_end < query_start
+              left = mid + 1
+            elsif interval_start > query_end
+              right = mid - 1
+            else
+              # Found overlap
+              return true
+            end
+          end
+
+          false
+        end
+
+        def empty?
+          @intervals.empty?
+        end
+
+        def intervals
+          build! unless @built
+          @intervals.dup
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/git/diff.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "set"
+require_relative "changed_lines"
+
+module Datadog
+  module CI
+    module Git
+      class Diff
+        FILE_CHANGE_REGEX = /^diff --git a\/(?<file>.+?) b\//.freeze
+        LINES_CHANGE_REGEX = /^@@ -\d+(?:,\d+)? \+(?<start>\d+)(?:,(?<count>\d+))? @@/.freeze
+
+        def initialize(changed_files: {})
+          @changed_files = changed_files # Hash of file_path => ChangedLines
+        end
+
+        # Check if any lines in the given range are changed for the specified file
+        def lines_changed?(file_path, start_line: nil, end_line: nil)
+          changed_lines = @changed_files[file_path]
+          unless changed_lines
+            Datadog.logger.debug { "No changes found for file: #{file_path}" }
+            return false
+          end
+
+          # If either start_line or end_line is nil, return true if file is present
+          return true if start_line.nil? || end_line.nil?
+
+          changed_lines.overlaps?(start_line, end_line)
+        end
+
+        def empty?
+          @changed_files.empty?
+        end
+
+        # for debug purposes
+        def size
+          @changed_files.size
+        end
+
+        # for debug purposes
+        def inspect
+          @changed_files.inspect
+        end
+
+        def self.parse_diff_output(output)
+          return new if output.nil? || output.empty?
+
+          changed_files = {}
+          current_file = nil
+
+          output.each_line do |line|
+            # Match lines like: diff --git a/foo/bar.rb b/foo/bar.rb
+            # This captures git changes on file level
+            match = FILE_CHANGE_REGEX.match(line)
+            if match && match[:file]
+              # this path here is already relative from the git root
+              changed_file = match[:file]
+
+              unless changed_file.nil? || changed_file.empty?
+                current_file = changed_file
+                changed_files[current_file] ||= ChangedLines.new
+              end
+
+              Datadog.logger.debug { "matched changed_file: #{changed_file} from git diff line: #{line}" }
+
+              next
+            end
+
+            # Match lines like: @@ -1,2 +3,4 @@
+            match = LINES_CHANGE_REGEX.match(line)
+            if match && match[:start] && current_file
+              start_line = match[:start].to_i
+
+              line_count = 1 # Default to 1 line if count not specified
+              line_count = match[:count].to_i if match[:count]
+
+              end_line = start_line + line_count - 1
+
+              changed_files[current_file].add_interval(start_line, end_line)
+
+              Datadog.logger.debug { "Added interval [#{start_line}, #{end_line}] for file: #{current_file}" }
+            end
+          end
+
+          new(changed_files: changed_files)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/git/local_repository.rb

@@ -8,6 +8,7 @@ require_relative "../ext/telemetry"
 require_relative "../utils/command"
 require_relative "base_branch_sha_detector"
 require_relative "cli"
+require_relative "diff"
 require_relative "telemetry"
 require_relative "user"
 
@@ -37,14 +38,22 @@ module Datadog
             # that root is a prefix of the path
             return "" if path.size < prefix_index
 
-            prefix_index += 1 if path[prefix_index] == File::SEPARATOR
-            res = path[prefix_index..]
+            # this means that the root is not a prefix of this path somehow
+            return "" if path[prefix_index] != File::SEPARATOR
+
+            res = path[prefix_index + 1..]
           else
             # prefix_to_root is a difference between the root path and the given path
-            if @prefix_to_root == ""
-              return path
-            elsif @prefix_to_root
-              return File.join(@prefix_to_root, path)
+            if defined?(@prefix_to_root)
+              # if path starts with ./ remove the dot before applying the optimization
+              # @type var path: String
+              path = path[1..] if path.start_with?("./")
+
+              if @prefix_to_root == ""
+                return path
+              elsif @prefix_to_root
+                return File.join(@prefix_to_root, path)
+              end
             end
 
             pathname = Pathname.new(File.expand_path(path))
@@ -308,10 +317,10 @@ module Datadog
           res
         end
 
-        # Returns a Set of normalized file paths changed since the given base_commit.
+        # Returns a Diff object with relative file paths for files that were changed since the given base_commit.
         # If base_commit is nil, returns nil. On error, returns nil.
-        def self.get_changed_files_from_diff(base_commit)
-          return nil if base_commit.nil?
+        def self.get_changes_since(base_commit)
+          return Diff.new if base_commit.nil?
 
           Datadog.logger.debug { "calculating git diff from base_commit: #{base_commit}" }
 
@@ -329,29 +338,14 @@ module Datadog
 
             Datadog.logger.debug { "git diff output: #{output}" }
 
-            return nil if output.nil?
-
-            # 2. Parse the output to extract which files changed
-            changed_files = Set.new
-            output.each_line do |line|
-              # Match lines like: diff --git a/foo/bar.rb b/foo/bar.rb
-              # This captures git changes on file level
-              match = /^diff --git a\/(?<file>.+?) b\//.match(line)
-              if match && match[:file]
-                changed_file = match[:file]
-                # Normalize to repo root
-                normalized_changed_file = relative_to_root(changed_file)
-                changed_files << normalized_changed_file unless normalized_changed_file.nil? || normalized_changed_file.empty?
-
-                Datadog.logger.debug { "matched changed_file: #{changed_file} from line: #{line}" }
-                Datadog.logger.debug { "normalized_changed_file: #{normalized_changed_file}" }
-              end
-            end
-            changed_files
+            return Diff.new if output.nil?
+
+            # 2. Parse the output using Git::Diff
+            Diff.parse_diff_output(output)
           rescue => e
             Telemetry.track_error(e, Ext::Telemetry::Command::DIFF)
             log_failure(e, "get changed files from diff")
-            nil
+            Diff.new
           end
         end
 

data/lib/datadog/ci/impacted_tests_detection/component.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
-
 require_relative "../ext/test"
 require_relative "../git/local_repository"
 
@@ -11,7 +9,7 @@ module Datadog
       class Component
         def initialize(enabled:)
           @enabled = enabled
-          @changed_files = Set.new
+          @git_diff = Git::Diff.new
         end
 
         def configure(library_settings, test_session)
@@ -29,21 +27,21 @@ module Datadog
             return
           end
 
-          changed_files = Git::LocalRepository.get_changed_files_from_diff(base_commit_sha)
-          if changed_files.nil?
+          git_diff = Git::LocalRepository.get_changes_since(base_commit_sha)
+          if git_diff.empty?
             Datadog.logger.debug { "Impacted tests detection disabled: could not get changed files" }
             @enabled = false
             return
           end
 
           Datadog.logger.debug do
-            "Impacted tests detection: found #{changed_files.size} changed files"
+            "Impacted tests detection: found #{git_diff.size} changed files"
           end
           Datadog.logger.debug do
-            "Impacted tests detection: changed files: #{changed_files.inspect}"
+            "Impacted tests detection: changed files: #{git_diff.inspect}"
           end
 
-          @changed_files = changed_files
+          @git_diff = git_diff
           @enabled = true
         end
 
@@ -57,7 +55,15 @@ module Datadog
           source_file = test_span.source_file
           return false if source_file.nil?
 
-          @changed_files.include?(source_file)
+          # convert to relative path without leading slash
+          # @type var source_file: String
+          source_file = source_file[1..] if source_file.start_with?("/")
+
+          result = @git_diff.lines_changed?(source_file, start_line: test_span.start_line, end_line: test_span.end_line)
+          Datadog.logger.debug do
+            "Impacted tests detection: test #{test_span.name} with source file #{source_file} is modified: #{result}"
+          end
+          result
         end
 
         def tag_modified_test(test_span)

data/lib/datadog/ci/test.rb

@@ -66,6 +66,22 @@ module Datadog
         get_tag(Ext::Test::TAG_TEST_SESSION_ID)
       end
 
+      # Returns the starting line number of the test in the source file.
+      # @return [Integer] the starting line number
+      # @return [nil] if the starting line is not available
+      def start_line
+        line = get_tag(Ext::Test::TAG_SOURCE_START)
+        line&.to_i
+      end
+
+      # Returns the ending line number of the test in the source file.
+      # @return [Integer] the ending line number
+      # @return [nil] if the ending line is not available
+      def end_line
+        line = get_tag(Ext::Test::TAG_SOURCE_END)
+        line&.to_i
+      end
+
       # Returns "true" if test span represents a retry.
       # @return [Boolean] true if this test is a retry, false otherwise.
       def is_retry?

data/lib/datadog/ci/utils/command.rb

@@ -92,6 +92,7 @@ module Datadog
         end
 
         def self.popen_with_stdin(command, stdin_data: nil, retries_left: OPEN_STDIN_RETRY_COUNT)
+          stdin = nil
           result = Open3.popen2e(*command)
           stdin = result.first
 
@@ -108,7 +109,7 @@ module Datadog
 
           result
         ensure
-          stdin.close
+          stdin&.close
         end
       end
     end

data/lib/datadog/ci/version.rb

@@ -4,7 +4,7 @@ module Datadog
   module CI
     module VERSION
       MAJOR = 1
-      MINOR = 18
+      MINOR = 19
       PATCH = 0
       PRE = nil
       BUILD = nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datadog-ci
 version: !ruby/object:Gem::Version
-  version: 1.18.0
+  version: 1.19.0
 platform: ruby
 authors:
 - Datadog, Inc.
@@ -196,7 +196,9 @@ files:
 - lib/datadog/ci/git/base_branch_sha_detection/guesser.rb
 - lib/datadog/ci/git/base_branch_sha_detection/merge_base_extractor.rb
 - lib/datadog/ci/git/base_branch_sha_detector.rb
+- lib/datadog/ci/git/changed_lines.rb
 - lib/datadog/ci/git/cli.rb
+- lib/datadog/ci/git/diff.rb
 - lib/datadog/ci/git/local_repository.rb
 - lib/datadog/ci/git/packfiles.rb
 - lib/datadog/ci/git/search_commits.rb