datadog-ci 1.13.0 → 1.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc134bf501f298c7803572320d3c82d93b64b0dbfc59b2d8219d42fd5fed24e6
-  data.tar.gz: 4714f599fef9eae12cb3dc99dfb17dc0431a9dfee7d9337da9fada542429c6c5
+  metadata.gz: c4507da31359cd997cb38e73190a4e6514ad8c69d86f9ce1b0c47fd7764e5dbd
+  data.tar.gz: 515872f426f95492fed9d1577f14bb7b1e447b0c011edf2bbfbb7ea23992d9cc
 SHA512:
-  metadata.gz: e54fa5b48885accd1c304fca925292470930f6c916e318c5a02eed24989d089e431b99aa56c3eab75f1ed4445c3ab8bd14b1783baa01c32e52bf3d0df5f3813b
-  data.tar.gz: 0f42913850d53736f3b058db3c7bea7e2f4cc0c5ccf511545d052afb142967084185075888511887185dbf5b82ee4b6931b7da76f91b91fe0c99760efb1b9574
+  metadata.gz: 3d0db7cb6a13a4c2f0b38b22c8dc2470a9ec760c38312c929a7b6224b118e18b279addf76901f7f094f60123a9e65d9afef06c15970bf01b247bc031bf80725d
+  data.tar.gz: f731bc7979a69ae63362d06d0313754bd825a22baf934cb682c1dc2a16a8584d0c3ece806f2bfb75d42924c09fff6296bf8c5c1be142afbe3ab7716bcb753815

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 ## [Unreleased]
 
+## [1.15.0] - 2025-03-25
+
+
+### Added
+
+* parallel tests gem support ([#299][])
+* implemented attempt to fix flow V2 ([#298][])
+
+### Fixed
+
+* Fix: prevent test impact analysis from skipping flaky tests that are attempted to be fixed ([#301][])
+* Fix git commit message extraction to extract multiline commit messages correctly ([#300][])
+
+## [1.14.0] - 2025-03-11
+
+### Added
+
+* Test impact analysis: add rails parallel testing support ([#294][])
+* Add parallel testing support to minitest framework ([#295][])
+
+### Changed
+
+* Test knapsack_pro v8 ([#292][])
+
 ## [1.13.0] - 2025-02-25
 
 ### Added
@@ -393,7 +417,9 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 
 - Ruby versions < 2.7 no longer supported ([#8][])
 
-[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.13.0...main
+[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.15.0...main
+[1.15.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.14.0...v1.15.0
+[1.14.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.13.0...v1.14.0
 [1.13.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.12.0...v1.13.0
 [1.12.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.11.0...v1.12.0
 [1.11.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.10.0...v1.11.0
@@ -564,4 +590,11 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 [#275]: https://github.com/DataDog/datadog-ci-rb/issues/275
 [#283]: https://github.com/DataDog/datadog-ci-rb/issues/283
 [#286]: https://github.com/DataDog/datadog-ci-rb/issues/286
-[#289]: https://github.com/DataDog/datadog-ci-rb/issues/289
+[#289]: https://github.com/DataDog/datadog-ci-rb/issues/289
+[#292]: https://github.com/DataDog/datadog-ci-rb/issues/292
+[#294]: https://github.com/DataDog/datadog-ci-rb/issues/294
+[#295]: https://github.com/DataDog/datadog-ci-rb/issues/295
+[#298]: https://github.com/DataDog/datadog-ci-rb/issues/298
+[#299]: https://github.com/DataDog/datadog-ci-rb/issues/299
+[#300]: https://github.com/DataDog/datadog-ci-rb/issues/300
+[#301]: https://github.com/DataDog/datadog-ci-rb/issues/301

data/lib/datadog/ci/configuration/components.rb

@@ -129,7 +129,8 @@ module Datadog
           @test_visibility = TestVisibility::Component.new(
             test_suite_level_visibility_enabled: !settings.ci.force_test_level_visibility,
             logical_test_session_name: settings.ci.test_session_name,
-            known_tests_client: build_known_tests_client(settings, test_visibility_api)
+            known_tests_client: build_known_tests_client(settings, test_visibility_api),
+            context_service_uri: settings.ci.test_visibility_drb_server_uri
           )
         end
 

data/lib/datadog/ci/configuration/settings.rb

@@ -134,6 +134,12 @@ module Datadog
                 o.default false
               end
 
+              # internal only
+              option :test_visibility_drb_server_uri do |o|
+                o.type :string, nilable: true
+                o.env CI::Ext::Settings::ENV_TEST_VISIBILITY_DRB_SERVER_URI
+              end
+
               define_method(:instrument) do |integration_name, options = {}, &block|
                 return unless enabled
 

data/lib/datadog/ci/contrib/minitest/helpers.rb

@@ -5,6 +5,32 @@ module Datadog
     module Contrib
       module Minitest
         module Helpers
+          def self.start_test_suite(klass)
+            method = klass.runnable_methods.first
+            return nil if method.nil?
+
+            test_suite_name = test_suite_name(klass, method)
+            source_file, line_number = extract_source_location_from_class(klass)
+
+            test_suite_tags = if source_file
+              {
+                CI::Ext::Test::TAG_SOURCE_FILE => (Git::LocalRepository.relative_to_root(source_file) if source_file),
+                CI::Ext::Test::TAG_SOURCE_START => line_number&.to_s
+              }
+            else
+              {}
+            end
+
+            test_visibility_component = Datadog.send(:components).test_visibility
+            test_suite = test_visibility_component.start_test_suite(
+              test_suite_name,
+              tags: test_suite_tags
+            )
+            test_suite&.set_expected_tests!(klass.runnable_methods)
+
+            test_suite
+          end
+
           def self.test_suite_name(klass, method_name)
             source_location = extract_source_location_from_class(klass)&.first
             # if we are in anonymous class, fallback to the method source location

data/lib/datadog/ci/contrib/minitest/runnable.rb

@@ -14,25 +14,7 @@ module Datadog
               return super unless datadog_configuration[:enabled]
               return super if Helpers.parallel?(self)
 
-              method = runnable_methods.first
-              return super if method.nil?
-
-              test_suite_name = Helpers.test_suite_name(self, method)
-              source_file, line_number = Helpers.extract_source_location_from_class(self)
-
-              test_suite_tags = if source_file
-                {
-                  CI::Ext::Test::TAG_SOURCE_FILE => (Git::LocalRepository.relative_to_root(source_file) if source_file),
-                  CI::Ext::Test::TAG_SOURCE_START => line_number&.to_s
-                }
-              else
-                {}
-              end
-
-              test_suite = test_visibility_component.start_test_suite(
-                test_suite_name,
-                tags: test_suite_tags
-              )
+              test_suite = Helpers.start_test_suite(self)
 
               results = super
               return results unless test_suite

data/lib/datadog/ci/contrib/minitest/runner.rb

@@ -9,8 +9,6 @@ module Datadog
     module Contrib
       module Minitest
         module Runner
-          DD_ESTIMATED_TESTS_PER_SUITE = 5
-
           def self.included(base)
             base.singleton_class.prepend(ClassMethods)
           end
@@ -21,15 +19,18 @@ module Datadog
 
               return unless datadog_configuration[:enabled]
 
-              # minitest does not store the total number of tests, so we can't pass it to the test session
-              # instead, we use the number of test suites * DD_ESTIMATED_TESTS_PER_SUITE as a rough estimate
+              tests_count = ::Minitest::Runnable.runnables.sum { |runnable| runnable.runnable_methods.size }
+
               test_visibility_component.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => datadog_integration.version.to_s
                 },
                 service: datadog_configuration[:service_name],
-                total_tests_count: (DD_ESTIMATED_TESTS_PER_SUITE * ::Minitest::Runnable.runnables.size).to_i
+                estimated_total_tests_count: tests_count,
+                # if minitest is being used with a parallel runner, then tests split will happen by example, not by test suite
+                # we need to always start/stop test suites in the parent process in this case
+                local_test_suites_mode: false
               )
               test_visibility_component.start_test_module(Ext::FRAMEWORK)
             end
@@ -43,6 +44,12 @@ module Datadog
                 result = super
               end
 
+              # get the current test suite and mark this method as done, so we can check if all tests were executed
+              # for this test suite
+              test_suite_name = Helpers.test_suite_name(klass, method_name)
+              test_suite = test_visibility_component.active_test_suite(test_suite_name)
+              test_suite&.expected_test_done!(method_name)
+
               result
             end
 

data/lib/datadog/ci/contrib/minitest/test.rb

@@ -22,14 +22,11 @@ module Datadog
               super
               return unless datadog_configuration[:enabled]
 
-              test_suite_name = Helpers.test_suite_name(self.class, name)
               if Helpers.parallel?(self.class)
-                test_suite_name = "#{test_suite_name} (#{name} concurrently)"
-
-                # for parallel execution we need to start a new test suite for each test
-                test_visibility_component.start_test_suite(test_suite_name)
+                Helpers.start_test_suite(self.class)
               end
 
+              test_suite_name = Helpers.test_suite_name(self.class, name)
               source_file, line_number = method(name).source_location
 
               test_span = test_visibility_component.trace_test(
@@ -56,10 +53,6 @@ module Datadog
               # remove failures if test passed at least once on retries or quarantined
               self.failures = [] if test_span.should_ignore_failures?
 
-              if Helpers.parallel?(self.class)
-                finish_with_result(test_span.test_suite, result_code)
-              end
-
               super
             end
 

data/lib/datadog/ci/contrib/parallel_tests/cli.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "../rspec/ext"
+
+module Datadog
+  module CI
+    module Contrib
+      module ParallelTests
+        module CLI
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          module InstanceMethods
+            def run_tests_in_parallel(num_processes, options)
+              # only rspec runner is supported for now
+              return super if @runner != ::ParallelTests::RSpec::Runner
+
+              begin
+                test_session = test_visibility_component.start_test_session(
+                  tags: {
+                    CI::Ext::Test::TAG_FRAMEWORK => CI::Contrib::RSpec::Ext::FRAMEWORK,
+                    CI::Ext::Test::TAG_FRAMEWORK_VERSION => datadog_extract_rspec_version
+                  },
+                  service: datadog_configuration[:service_name],
+                  estimated_total_tests_count: 10_000, # temporary value, updated by child processes
+                  distributed: true
+                )
+                test_module = test_visibility_component.start_test_module("rspec")
+
+                options[:env] ||= {}
+                options[:env][CI::Ext::Settings::ENV_TEST_VISIBILITY_DRB_SERVER_URI] = test_visibility_component.context_service_uri
+
+                super
+              ensure
+                test_module&.finish
+                test_session&.finish
+              end
+            end
+
+            def any_test_failed?(test_results)
+              res = super
+
+              test_session = test_visibility_component.active_test_session
+              test_module = test_visibility_component.active_test_module
+              if res
+                test_module&.failed!
+                test_session&.failed!
+              else
+                test_module&.passed!
+                test_session&.passed!
+              end
+
+              res
+            end
+
+            def datadog_extract_rspec_version
+              # Try to find either 'rspec' or 'rspec-core' gem
+              if Gem.loaded_specs["rspec"]
+                Gem.loaded_specs["rspec"].version.to_s
+              elsif Gem.loaded_specs["rspec-core"]
+                Gem.loaded_specs["rspec-core"].version.to_s
+              else
+                "0.0.0"
+              end
+            rescue => e
+              Datadog.logger.debug("Error extracting RSpec version: #{e.class.name} - #{e.message}")
+              "0.0.0"
+            end
+
+            def datadog_configuration
+              Datadog.configuration.ci[:paralleltests]
+            end
+
+            def test_visibility_component
+              Datadog.send(:components).test_visibility
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/parallel_tests/configuration/settings.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative "../ext"
+require_relative "../../settings"
+require_relative "../../../utils/configuration"
+
+module Datadog
+  module CI
+    module Contrib
+      module ParallelTests
+        module Configuration
+          # Custom settings for the ParallelTests integration
+          # @public_api
+          class Settings < Datadog::CI::Contrib::Settings
+            option :enabled do |o|
+              o.type :bool
+              o.env Ext::ENV_ENABLED
+              o.default true
+            end
+
+            option :service_name do |o|
+              o.type :string
+              o.default do
+                Utils::Configuration.fetch_service_name(Ext::DEFAULT_SERVICE_NAME)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/parallel_tests/ext.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module ParallelTests
+        # Datadog ParallelTests integration constants
+        module Ext
+          ENV_ENABLED = "DD_TRACE_PARALLEL_TESTS_ENABLED"
+
+          DEFAULT_SERVICE_NAME = "parallel_tests"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/parallel_tests/integration.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "../integration"
+require_relative "configuration/settings"
+require_relative "patcher"
+
+module Datadog
+  module CI
+    module Contrib
+      module ParallelTests
+        # Description of ParallelTests integration
+        class Integration < Datadog::CI::Contrib::Integration
+          MINIMUM_VERSION = Gem::Version.new("4.0")
+
+          def version
+            Gem.loaded_specs["parallel_tests"]&.version
+          end
+
+          def loaded?
+            !defined?(::ParallelTests).nil? && !defined?(::ParallelTests::CLI).nil?
+          end
+
+          def compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          def late_instrument?
+            false
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/parallel_tests/patcher.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative "cli"
+require_relative "integration"
+require_relative "../patcher"
+
+module Datadog
+  module CI
+    module Contrib
+      module ParallelTests
+        # Patcher enables patching of parallel_tests module
+        module Patcher
+          include Datadog::CI::Contrib::Patcher
+
+          module_function
+
+          def patch
+            ::ParallelTests::CLI.include(CLI)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/rspec/example.rb

@@ -97,6 +97,13 @@ module Datadog
               )
             end
 
+            def datadog_fqn_test_id
+              @datadog_fqn_test_id ||= Utils::TestRun.datadog_test_id(
+                datadog_test_name,
+                datadog_test_suite_name
+              )
+            end
+
             def datadog_unskippable?
               !!metadata[CI::Ext::Test::ITR_UNSKIPPABLE_OPTION]
             end

data/lib/datadog/ci/contrib/rspec/example_group.rb

@@ -2,6 +2,7 @@
 
 require_relative "../../ext/test"
 require_relative "ext"
+require_relative "helpers"
 
 module Datadog
   module CI
@@ -28,13 +29,17 @@ module Datadog
               return super unless top_level?
 
               suite_name = "#{description} at #{file_path}"
-              test_suite = test_visibility_component&.start_test_suite(
-                suite_name,
-                tags: {
-                  CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(metadata[:file_path]),
-                  CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s
-                }
-              )
+              suite_tags = {
+                CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(metadata[:file_path]),
+                CI::Ext::Test::TAG_SOURCE_START => metadata[:line_number].to_s
+              }
+
+              test_suite =
+                test_visibility_component&.start_test_suite(
+                  suite_name,
+                  tags: suite_tags,
+                  service: datadog_configuration[:service_name]
+                )
 
               success = super
               return success unless test_suite
@@ -56,7 +61,8 @@ module Datadog
 
             def all_examples_skipped_by_datadog?
               descendant_filtered_examples.all? do |example|
-                !example.datadog_unskippable? && test_optimisation_component&.skippable?(example)
+                !example.datadog_unskippable? && test_optimisation_component&.skippable?(example.datadog_test_id) &&
+                  !test_management_component&.attempt_to_fix?(example.datadog_fqn_test_id)
               end
             end
 
@@ -71,6 +77,10 @@ module Datadog
             def test_optimisation_component
               Datadog.send(:components).test_optimisation
             end
+
+            def test_management_component
+              Datadog.send(:components).test_management
+            end
           end
         end
       end

data/lib/datadog/ci/contrib/rspec/helpers.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module RSpec
+        # Helper methods for RSpec instrumentation
+        module Helpers
+          module_function
+
+          def parallel_tests?
+            !!ENV.fetch("TEST_ENV_NUMBER", nil) && !!ENV.fetch("PARALLEL_TEST_GROUPS", nil)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/rspec/runner.rb

@@ -3,6 +3,7 @@
 require_relative "../../ext/test"
 require_relative "../instrumentation"
 require_relative "ext"
+require_relative "helpers"
 
 module Datadog
   module CI
@@ -25,13 +26,14 @@ module Datadog
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => datadog_integration.version.to_s
                 },
                 service: datadog_configuration[:service_name],
-                total_tests_count: ::RSpec.world.example_count
+                estimated_total_tests_count: ::RSpec.world.example_count
               )
 
               test_module = test_visibility_component.start_test_module(Ext::FRAMEWORK)
 
               result = super
               return result unless test_module && test_session
+              return result if Helpers.parallel_tests?
 
               if result != 0
                 test_module.failed!

data/lib/datadog/ci/ext/settings.rb

@@ -22,6 +22,7 @@ module Datadog
         ENV_TEST_SESSION_NAME = "DD_TEST_SESSION_NAME"
         ENV_TEST_MANAGEMENT_ENABLED = "DD_TEST_MANAGEMENT_ENABLED"
         ENV_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES = "DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES"
+        ENV_TEST_VISIBILITY_DRB_SERVER_URI = "DD_TEST_VISIBILITY_DRB_SERVER_URI"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = %w[

data/lib/datadog/ci/ext/test.rb

@@ -86,6 +86,26 @@ module Datadog
           TAG_TEST_MANAGEMENT_QUARANTINE = "_dd.library_capabilities.test_management.quarantine"
           TAG_TEST_MANAGEMENT_DISABLE = "_dd.library_capabilities.test_management.disable"
           TAG_TEST_MANAGEMENT_ATTEMPT_TO_FIX = "_dd.library_capabilities.test_management.attempt_to_fix"
+
+          # Version numbers for library capabilities
+          module Versions
+            TEST_IMPACT_ANALYSIS_VERSION = "1"
+            EARLY_FLAKE_DETECTION_VERSION = "1"
+            AUTO_TEST_RETRIES_VERSION = "1"
+            TEST_MANAGEMENT_QUARANTINE_VERSION = "1"
+            TEST_MANAGEMENT_DISABLE_VERSION = "1"
+            TEST_MANAGEMENT_ATTEMPT_TO_FIX_VERSION = "2"
+          end
+
+          # Map of capabilities to their versions
+          CAPABILITY_VERSIONS = {
+            TAG_TEST_IMPACT_ANALYSIS => Versions::TEST_IMPACT_ANALYSIS_VERSION,
+            TAG_EARLY_FLAKE_DETECTION => Versions::EARLY_FLAKE_DETECTION_VERSION,
+            TAG_AUTO_TEST_RETRIES => Versions::AUTO_TEST_RETRIES_VERSION,
+            TAG_TEST_MANAGEMENT_QUARANTINE => Versions::TEST_MANAGEMENT_QUARANTINE_VERSION,
+            TAG_TEST_MANAGEMENT_DISABLE => Versions::TEST_MANAGEMENT_DISABLE_VERSION,
+            TAG_TEST_MANAGEMENT_ATTEMPT_TO_FIX => Versions::TEST_MANAGEMENT_ATTEMPT_TO_FIX_VERSION
+          }.freeze
         end
 
         # internal APM tag to mark a span as a test span

data/lib/datadog/ci/git/local_repository.rb

@@ -142,7 +142,7 @@ module Datadog
         end
 
         def self.git_commit_message
-          exec_git_command("git show -s --format=%s")
+          exec_git_command("git log -n 1 --format=%B")
         rescue => e
           log_failure(e, "git commit message")
           nil

data/lib/datadog/ci/git/tree_uploader.rb

@@ -27,6 +27,11 @@ module Datadog
             return
           end
 
+          if test_visibility_component.client_process?
+            Datadog.logger.debug("Test visibility component is running in client process, aborting git upload")
+            return
+          end
+
           Datadog.logger.debug { "Uploading git tree for repository #{repository_url}" }
 
           latest_commits = LocalRepository.git_commits
@@ -91,6 +96,10 @@ module Datadog
             backend_commits.include?(commit)
           end
         end
+
+        def test_visibility_component
+          Datadog.send(:components).test_visibility
+        end
       end
     end
   end

data/lib/datadog/ci/readonly_test_module.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "test_module"
+
+module Datadog
+  module CI
+    # @internal_api
+    class ReadonlyTestModule < TestModule
+      def initialize(test_module)
+        @id = test_module.id
+        @name = test_module.name
+      end
+      attr_reader :id, :name
+
+      def finish
+        raise "ReadonlyTestModule cannot be finished"
+      end
+
+      def set_tag(key, value)
+        raise "ReadonlyTestModule cannot be modified"
+      end
+
+      def set_metric(key, value)
+        raise "ReadonlyTestModule cannot be modified"
+      end
+    end
+  end
+end

data/lib/datadog/ci/readonly_test_session.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "test_session"
+
+module Datadog
+  module CI
+    # @internal_api
+    class ReadonlyTestSession < TestSession
+      def initialize(test_session)
+        @id = test_session.id
+        @name = test_session.name
+        @inheritable_tags = test_session.inheritable_tags
+        @service = test_session.service
+      end
+
+      attr_reader :id, :name, :inheritable_tags, :service
+
+      def finish
+        raise "ReadonlyTestSession cannot be finished"
+      end
+
+      def set_tag(key, value)
+        raise "ReadonlyTestSession cannot be modified"
+      end
+
+      def set_metric(key, value)
+        raise "ReadonlyTestSession cannot be modified"
+      end
+    end
+  end
+end