datadog-ci 1.7.0 → 1.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4cb94bf0ef94f54289e38a26649d24eeaaca99ba91229f5ebb096c046bdc582
-  data.tar.gz: 9a88847a58121a1516b95fc1abac1fa89d4a1bab5c4593f932fc804410ee8a57
+  metadata.gz: b5ee5645ee9ec6c07af9660a2742402d617965668a00e48764839d6b7ca8470e
+  data.tar.gz: 878ee49cc336a2d8d37e718c09a6f17b949cc149c8f6a57162410ff2da9ccfea
 SHA512:
-  metadata.gz: 7830a7b52821efbea8e0f86e878588ef96718e845fd7a33575a4a5e92d350e26e36bb4b78f7ce78b58ff59654c62973c4eca58f21c63b958d44f4d8feff122cc
-  data.tar.gz: da8d30e84f9dea71eb498cccb6e03cb092e27e773008dc7533188020ee5cb59de774e4d2be32821e3fd82e28f4ccb426572695fd532dc7f7fff6479a392bca85
+  metadata.gz: 4fc31d4106750af2d5cc556164459e24c6782e7ade09abf57b0a3a48320552aa74f9b42488a40df256b0562f304883249b86c44b16eef59cce00af4c15f00aa9
+  data.tar.gz: fa12a596824ad2963bcd394c8a634db54300e4fab62f482c17bad0210a7a6b159a48355a070dd65a413412225ab5f570d125c547390d46f946a4fd6389244222

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 ## [Unreleased]
 
+## [1.8.1] - 2024-10-18
+
+
+### Fixed
+* Make --spec-path option available to skipped-tests-estimate cli command ([#250][])
+
+## [1.8.0] - 2024-10-17
+
+### Added
+* Add command line tool to compute a percentage of skippable tests for RSpec ([#194][])
+
+### Changed
+* Bump gem datadog dependency to 2.4 and update test dependencies ([#248][])
+* Optimise LocalRepository.relative_to_root helper to make test impact analysis faster ([#244][])
+* Retry HTTP requests on 429 and 5xx responses ([#243][])
+* Use correct monotonic clock time if Timecop.mock_process_clock is set ([#242][])
+
 ## [1.7.0] - 2024-09-25
 
 ### Added
@@ -339,7 +356,9 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 
 - Ruby versions < 2.7 no longer supported ([#8][])
 
-[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.7.0...main
+[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.8.1...main
+[1.8.1]: https://github.com/DataDog/datadog-ci-rb/compare/v1.8.0...v1.8.1
+[1.8.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.7.0...v1.8.0
 [1.7.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.6.0...v1.7.0
 [1.6.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.5.0...v1.6.0
 [1.5.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.4.1...v1.5.0
@@ -459,6 +478,7 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 [#189]: https://github.com/DataDog/datadog-ci-rb/issues/189
 [#190]: https://github.com/DataDog/datadog-ci-rb/issues/190
 [#193]: https://github.com/DataDog/datadog-ci-rb/issues/193
+[#194]: https://github.com/DataDog/datadog-ci-rb/issues/194
 [#197]: https://github.com/DataDog/datadog-ci-rb/issues/197
 [#200]: https://github.com/DataDog/datadog-ci-rb/issues/200
 [#201]: https://github.com/DataDog/datadog-ci-rb/issues/201
@@ -488,4 +508,9 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 [#236]: https://github.com/DataDog/datadog-ci-rb/issues/236
 [#238]: https://github.com/DataDog/datadog-ci-rb/issues/238
 [#239]: https://github.com/DataDog/datadog-ci-rb/issues/239
-[#240]: https://github.com/DataDog/datadog-ci-rb/issues/240
+[#240]: https://github.com/DataDog/datadog-ci-rb/issues/240
+[#242]: https://github.com/DataDog/datadog-ci-rb/issues/242
+[#243]: https://github.com/DataDog/datadog-ci-rb/issues/243
+[#244]: https://github.com/DataDog/datadog-ci-rb/issues/244
+[#248]: https://github.com/DataDog/datadog-ci-rb/issues/248
+[#250]: https://github.com/DataDog/datadog-ci-rb/issues/250

data/exe/ddcirb

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "datadog/ci/cli/cli"
+
+Datadog::CI::CLI.exec(ARGV.first)

data/lib/datadog/ci/cli/cli.rb

@@ -0,0 +1,24 @@
+require "datadog"
+require "datadog/ci"
+
+require_relative "command/skippable_tests_percentage"
+require_relative "command/skippable_tests_percentage_estimate"
+
+module Datadog
+  module CI
+    module CLI
+      def self.exec(action)
+        case action
+        when "skipped-tests", "skippable-tests"
+          Command::SkippableTestsPercentage.new.exec
+        when "skipped-tests-estimate", "skippable-tests-estimate"
+          Command::SkippableTestsPercentageEstimate.new.exec
+        else
+          puts("Usage: bundle exec ddcirb [command] [options]. Available commands:")
+          puts("  skippable-tests - calculates the exact percentage of skipped tests and prints it to stdout or file")
+          puts("  skippable-tests-estimate - estimates the percentage of skipped tests and prints it to stdout or file")
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/cli/command/base.rb

@@ -0,0 +1,59 @@
+require "optparse"
+
+module Datadog
+  module CI
+    module CLI
+      module Command
+        class Base
+          def exec
+            action = build_action
+            result = action&.call
+
+            validate!(action)
+            output(result)
+          end
+
+          private
+
+          def build_action
+          end
+
+          def options
+            return @options if defined?(@options)
+
+            ddcirb_options = {}
+            OptionParser.new do |opts|
+              opts.banner = "Usage: bundle exec ddcirb [command] [options]\n Available commands: skippable-tests, skippable-tests-estimate"
+
+              opts.on("-f", "--file FILENAME", "Output result to file FILENAME")
+              opts.on("--verbose", "Verbose output to stdout")
+              opts.on("--spec-path=[SPEC_PATH]", "Relative path to the spec directory, example: spec")
+
+              command_options(opts)
+            end.parse!(into: ddcirb_options)
+
+            @options = ddcirb_options
+          end
+
+          def command_options(opts)
+          end
+
+          def validate!(action)
+            if action.nil? || action.failed
+              Datadog.logger.error("ddcirb failed, exiting")
+              Kernel.exit(1)
+            end
+          end
+
+          def output(result)
+            if options[:file]
+              File.write(options[:file], result)
+            else
+              print(result)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/cli/command/skippable_tests_percentage.rb

@@ -0,0 +1,26 @@
+require_relative "base"
+require_relative "../../test_optimisation/skippable_percentage/calculator"
+
+module Datadog
+  module CI
+    module CLI
+      module Command
+        class SkippableTestsPercentage < Base
+          private
+
+          def build_action
+            ::Datadog::CI::TestOptimisation::SkippablePercentage::Calculator.new(
+              rspec_cli_options: (options[:"rspec-opts"] || "").split,
+              verbose: !options[:verbose].nil?,
+              spec_path: options[:"spec-path"] || "spec"
+            )
+          end
+
+          def command_options(opts)
+            opts.on("--rspec-opts=[OPTIONS]", "Command line options to pass to RSpec")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/cli/command/skippable_tests_percentage_estimate.rb

@@ -0,0 +1,21 @@
+require_relative "base"
+require_relative "../../test_optimisation/skippable_percentage/estimator"
+
+module Datadog
+  module CI
+    module CLI
+      module Command
+        class SkippableTestsPercentageEstimate < Base
+          private
+
+          def build_action
+            ::Datadog::CI::TestOptimisation::SkippablePercentage::Estimator.new(
+              verbose: !options[:verbose].nil?,
+              spec_path: options[:"spec-path"] || "spec"
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/configuration/components.rb

@@ -19,6 +19,7 @@ require_relative "../test_visibility/serializers/factories/test_level"
 require_relative "../test_visibility/serializers/factories/test_suite_level"
 require_relative "../test_visibility/transport"
 require_relative "../transport/adapters/telemetry_webmock_safe_adapter"
+require_relative "../test_visibility/null_transport"
 require_relative "../transport/api/builder"
 require_relative "../utils/parsing"
 require_relative "../utils/test_run"
@@ -79,16 +80,8 @@ module Datadog
           # startup logs are useless for test visibility and create noise
           settings.diagnostics.startup_logs.enabled = false
 
-          # When timecop is present, Time.now is mocked and .now_without_mock_time is added on Time to
-          # get the current time without the mock.
-          if timecop?
-            settings.time_now_provider = -> do
-              Time.now_without_mock_time
-            rescue NoMethodError
-              # fallback to normal Time.now if Time.now_without_mock_time is not defined for any reason
-              Time.now
-            end
-          end
+          # timecop configuration
+          configure_time_providers(settings)
 
           # Configure Datadog::Tracing module
 
@@ -181,7 +174,6 @@ module Datadog
               # Tests are running without CI visibility enabled
               settings.ci.enabled = false
             end
-
           else
             Datadog.logger.debug("CI visibility configured to use agent transport via EVP proxy")
 
@@ -203,6 +195,9 @@ module Datadog
         end
 
         def build_tracing_transport(settings, api)
+          # NullTransport ignores traces
+          return TestVisibility::NullTransport.new if settings.ci.discard_traces
+          # nil means that default legacy APM transport will be used (only for very old Datadog Agent versions)
           return nil if api.nil?
 
           TestVisibility::Transport.new(
@@ -213,7 +208,8 @@ module Datadog
         end
 
         def build_coverage_writer(settings, api)
-          return nil if api.nil?
+          # nil means that coverage event will be ignored
+          return nil if api.nil? || settings.ci.discard_traces
 
           TestOptimisation::Coverage::Writer.new(
             transport: TestOptimisation::Coverage::Transport.new(api: api)
@@ -294,6 +290,29 @@ module Datadog
           end
         end
 
+        # When timecop is present:
+        # - Time.now is mocked and .now_without_mock_time is added on Time to get the current time without the mock.
+        # - Process.clock_gettime is mocked and .clock_gettime_without_mock is added on Process to get the monotonic time without the mock.
+        def configure_time_providers(settings)
+          return unless timecop?
+
+          settings.time_now_provider = -> do
+            Time.now_without_mock_time
+          rescue NoMethodError
+            # fallback to normal Time.now if Time.now_without_mock_time is not defined for any reason
+            Time.now
+          end
+
+          if defined?(Process.clock_gettime_without_mock)
+            settings.get_time_provider = ->(unit = :float_second) do
+              ::Process.clock_gettime_without_mock(::Process::CLOCK_MONOTONIC, unit)
+            rescue NoMethodError
+              # fallback to normal Process.clock_gettime if Process.clock_gettime_without_mock is not defined for any reason
+              Process.clock_gettime(::Process::CLOCK_MONOTONIC, unit)
+            end
+          end
+        end
+
         def timecop?
           Gem.loaded_specs.key?("timecop") || !!defined?(Timecop)
         end

data/lib/datadog/ci/configuration/settings.rb

@@ -117,6 +117,12 @@ module Datadog
                 o.default true
               end
 
+              # internal only
+              option :discard_traces do |o|
+                o.type :bool
+                o.default false
+              end
+
               define_method(:instrument) do |integration_name, options = {}, &block|
                 return unless enabled
 

data/lib/datadog/ci/contrib/rspec/configuration/settings.rb

@@ -24,6 +24,12 @@ module Datadog
                 Utils::Configuration.fetch_service_name(Ext::DEFAULT_SERVICE_NAME)
               end
             end
+
+            # internal only
+            option :dry_run_enabled do |o|
+              o.type :bool
+              o.default false
+            end
           end
         end
       end

data/lib/datadog/ci/contrib/rspec/example.rb

@@ -17,7 +17,7 @@ module Datadog
 
           module InstanceMethods
             def run(*args)
-              return super if ::RSpec.configuration.dry_run?
+              return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled]
               return super unless datadog_configuration[:enabled]
 
               test_name = full_description.strip

data/lib/datadog/ci/contrib/rspec/example_group.rb

@@ -16,7 +16,7 @@ module Datadog
           # Instance methods for configuration
           module ClassMethods
             def run(reporter = ::RSpec::Core::NullReporter)
-              return super if ::RSpec.configuration.dry_run?
+              return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled]
               return super unless datadog_configuration[:enabled]
               return super unless top_level?
 

data/lib/datadog/ci/contrib/rspec/knapsack_pro/runner.rb

@@ -15,7 +15,7 @@ module Datadog
 
             module InstanceMethods
               def knapsack__run_specs(*args)
-                return super if ::RSpec.configuration.dry_run?
+                return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled]
                 return super unless datadog_configuration[:enabled]
 
                 test_session = test_visibility_component.start_test_session(

data/lib/datadog/ci/contrib/rspec/runner.rb

@@ -15,7 +15,7 @@ module Datadog
 
           module InstanceMethods
             def run_specs(*args)
-              return super if ::RSpec.configuration.dry_run?
+              return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled]
               return super unless datadog_configuration[:enabled]
 
               test_session = test_visibility_component.start_test_session(

data/lib/datadog/ci/ext/transport.rb

@@ -12,6 +12,7 @@ module Datadog
         HEADER_CONTENT_ENCODING = "Content-Encoding"
         HEADER_EVP_SUBDOMAIN = "X-Datadog-EVP-Subdomain"
         HEADER_CONTAINER_ID = "Datadog-Container-ID"
+        HEADER_RATELIMIT_RESET = "X-RateLimit-Reset"
 
         EVP_PROXY_V2_PATH_PREFIX = "/evp_proxy/v2/"
         EVP_PROXY_V4_PATH_PREFIX = "/evp_proxy/v4/"

data/lib/datadog/ci/git/local_repository.rb

@@ -30,16 +30,44 @@ module Datadog
           @root = git_root || Dir.pwd
         end
 
+        # ATTENTION: this function is running in a hot path
+        # and should be optimized for performance
         def self.relative_to_root(path)
           return "" if path.nil?
 
           root_path = root
           return path if root_path.nil?
 
-          path = Pathname.new(File.expand_path(path))
-          root_path = Pathname.new(root_path)
+          if File.absolute_path?(path)
+            # prefix_index is where the root path ends in the given path
+            prefix_index = root_path.size
+
+            # impossible case - absolute paths are returned from code coverage tool that always checks
+            # that root is a prefix of the path
+            return "" if path.size < prefix_index
+
+            prefix_index += 1 if path[prefix_index] == File::SEPARATOR
+            res = path[prefix_index..]
+          else
+            # prefix_to_root is a difference between the root path and the given path
+            if @prefix_to_root == ""
+              return path
+            elsif @prefix_to_root
+              return File.join(@prefix_to_root, path)
+            end
+
+            pathname = Pathname.new(File.expand_path(path))
+            root_path = Pathname.new(root_path)
+
+            # relative_path_from is an expensive function
+            res = pathname.relative_path_from(root_path).to_s
+
+            unless defined?(@prefix_to_root)
+              @prefix_to_root = res&.gsub(path, "") if res.end_with?(path)
+            end
+          end
 
-          path.relative_path_from(root_path).to_s
+          res || ""
         end
 
         def self.repository_name

data/lib/datadog/ci/test.rb

@@ -76,7 +76,7 @@ module Datadog
       # - tests that read files from disk
       # - tests that make network requests
       # - tests that call external processes
-      # - tests that use forking or threading
+      # - tests that use forking
       #
       # @return [void]
       def itr_unskippable!

data/lib/datadog/ci/test_optimisation/component.rb

@@ -25,7 +25,9 @@ module Datadog
       class Component
         include Core::Utils::Forking
 
-        attr_reader :correlation_id, :skippable_tests, :skipped_tests_count
+        attr_reader :correlation_id, :skippable_tests, :skippable_tests_fetch_error,
+          :skipped_tests_count, :total_tests_count,
+          :enabled, :test_skipping_enabled, :code_coverage_enabled
 
         def initialize(
           dd_env:,
@@ -58,7 +60,9 @@ module Datadog
           @correlation_id = nil
           @skippable_tests = Set.new
 
+          @total_tests_count = 0
           @skipped_tests_count = 0
+
           @mutex = Mutex.new
 
           Datadog.logger.debug("TestOptimisation initialized with enabled: #{@enabled}")
@@ -159,14 +163,16 @@ module Datadog
         end
 
         def count_skipped_test(test)
-          return if !test.skipped? || !test.skipped_by_itr?
+          @mutex.synchronize do
+            @total_tests_count += 1
 
-          if forked?
-            Datadog.logger.warn { "Intelligent test runner is not supported for forking test runners yet" }
-            return
-          end
+            return if !test.skipped? || !test.skipped_by_itr?
+
+            if forked?
+              Datadog.logger.warn { "ITR is not supported for forking test runners yet" }
+              return
+            end
 
-          @mutex.synchronize do
             Telemetry.itr_skipped
 
             @skipped_tests_count += 1
@@ -183,6 +189,10 @@ module Datadog
           test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_COUNT, @skipped_tests_count)
         end
 
+        def skippable_tests_count
+          skippable_tests.count
+        end
+
         def shutdown!
           @coverage_writer&.stop
         end
@@ -229,6 +239,7 @@ module Datadog
           skippable_response =
             Skippable.new(api: @api, dd_env: @dd_env, config_tags: @config_tags)
               .fetch_skippable_tests(test_session)
+          @skippable_tests_fetch_error = skippable_response.error_message unless skippable_response.ok?
 
           @correlation_id = skippable_response.correlation_id
           @skippable_tests = skippable_response.tests

data/lib/datadog/ci/test_optimisation/skippable.rb

@@ -42,6 +42,12 @@ module Datadog
             res
           end
 
+          def error_message
+            return nil if ok?
+
+            "Status code: #{@http_response&.code}, response: #{@http_response&.payload}"
+          end
+
           private
 
           def payload

data/lib/datadog/ci/test_optimisation/skippable_percentage/base.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module TestOptimisation
+      module SkippablePercentage
+        class Base
+          attr_reader :failed
+
+          def initialize(verbose: false, spec_path: "spec")
+            @verbose = verbose
+            @spec_path = spec_path
+            @failed = false
+
+            log("Spec path: #{@spec_path}")
+            error!("Spec path is not a directory: #{@spec_path}") if !File.directory?(@spec_path)
+          end
+
+          def call
+            0.0
+          end
+
+          private
+
+          def validate_test_optimisation_state!
+            unless test_optimisation.enabled
+              error!("ITR wasn't enabled, check the environment variables (DD_SERVICE, DD_ENV)")
+            end
+
+            if test_optimisation.skippable_tests_fetch_error
+              error!("Skippable tests couldn't be fetched, error: #{test_optimisation.skippable_tests_fetch_error}")
+            end
+          end
+
+          def log(message)
+            Datadog.logger.info(message) if @verbose
+          end
+
+          def error!(message)
+            Datadog.logger.error(message)
+            @failed = true
+          end
+
+          def test_optimisation
+            Datadog.send(:components).test_optimisation
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimisation/skippable_percentage/calculator.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Datadog
+  module CI
+    module TestOptimisation
+      module SkippablePercentage
+        # This class calculates the percentage of tests that are going to be skipped in the next run
+        # without actually running the tests.
+        #
+        # It is useful to determine the number of parallel jobs that are required for the CI pipeline.
+        #
+        # NOTE: Only RSpec is supported at the moment.
+        class Calculator < Base
+          def initialize(rspec_cli_options: [], verbose: false, spec_path: "spec")
+            super(verbose: verbose, spec_path: spec_path)
+
+            @rspec_cli_options = rspec_cli_options || []
+          end
+
+          def call
+            return 0.0 if @failed
+
+            require_rspec!
+            return 0.0 if @failed
+
+            configure_datadog
+
+            exit_code = dry_run
+            if exit_code != 0
+              Datadog.logger.error("RSpec dry-run failed with exit code #{exit_code}")
+              @failed = true
+              return 0.0
+            end
+
+            log("Total tests count: #{test_optimisation.total_tests_count}")
+            log("Skipped tests count: #{test_optimisation.skipped_tests_count}")
+            validate_test_optimisation_state!
+
+            (test_optimisation.skipped_tests_count.to_f / test_optimisation.total_tests_count.to_f).floor(2)
+          end
+
+          private
+
+          def require_rspec!
+            require "rspec/core"
+          rescue LoadError
+            Datadog.logger.error("RSpec is not installed, currently this functionality is only supported for RSpec.")
+            @failed = true
+          end
+
+          def configure_datadog
+            Datadog.configure do |c|
+              c.ci.enabled = true
+              c.ci.itr_enabled = true
+              c.ci.retry_failed_tests_enabled = false
+              c.ci.retry_new_tests_enabled = false
+              c.ci.discard_traces = true
+              c.ci.instrument :rspec, dry_run_enabled: true
+              c.tracing.enabled = true
+            end
+          end
+
+          def dry_run
+            cli_options_array = @rspec_cli_options + ["--dry-run", @spec_path]
+
+            rspec_config_options = ::RSpec::Core::ConfigurationOptions.new(cli_options_array)
+            devnull = File.new("/dev/null", "w")
+            out = @verbose ? $stdout : devnull
+            err = @verbose ? $stderr : devnull
+
+            ::RSpec::Core::Runner.new(rspec_config_options).run(out, err)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimisation/skippable_percentage/estimator.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Datadog
+  module CI
+    module TestOptimisation
+      module SkippablePercentage
+        # This class estimates the percentage of tests that are going to be skipped in the next run
+        # without actually running the tests. This estimate is very rough:
+        #
+        # - it counts the number of lines that start with "it" or "scenario" in the spec files, which could be inaccurate
+        #   if you use shared examples
+        # - it only counts the number of tests that could be skipped, this does not mean that they will be actually skipped:
+        #   if in this commit you replaced all the tests in your test suite with new ones, all the tests would be run (but
+        #   this is highly unlikely)
+        #
+        # It is useful to determine the number of parallel jobs that are required for the CI pipeline.
+        #
+        # NOTE: Only RSpec is supported at the moment.
+        class Estimator < Base
+          def initialize(verbose: false, spec_path: "spec")
+            super
+          end
+
+          def call
+            return 0.0 if @failed
+
+            Datadog.configure do |c|
+              c.ci.enabled = true
+              c.ci.itr_enabled = true
+              c.ci.retry_failed_tests_enabled = false
+              c.ci.retry_new_tests_enabled = false
+              c.ci.discard_traces = true
+              c.tracing.enabled = true
+            end
+
+            spec_files = Dir["#{@spec_path}/**/*_spec.rb"]
+            estimated_tests_count = spec_files.sum do |file|
+              content = File.read(file)
+              content.scan(/(^\s*it\s+)|(^\s*scenario\s+)/).size
+            end
+
+            # starting and finishing a test session is required to get the skippable tests response
+            Datadog::CI.start_test_session(total_tests_count: estimated_tests_count)&.finish
+            skippable_tests_count = test_optimisation.skippable_tests_count
+
+            log("Estimated tests count: #{estimated_tests_count}")
+            log("Skippable tests count: #{skippable_tests_count}")
+            validate_test_optimisation_state!
+
+            [(skippable_tests_count.to_f / estimated_tests_count).floor(2), 0.99].min || 0.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_visibility/null_transport.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module TestVisibility
+      class NullTransport
+        def initialize
+        end
+
+        def send_traces(traces)
+          []
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/transport/http.rb

@@ -23,6 +23,7 @@ module Datadog
         DEFAULT_TIMEOUT = 30
         MAX_RETRIES = 3
         INITIAL_BACKOFF = 1
+        MAX_BACKOFF = 30
 
         def initialize(host:, port:, timeout: DEFAULT_TIMEOUT, ssl: true, compress: false)
           @host = host
@@ -78,21 +79,48 @@ module Datadog
         private
 
         def perform_http_call(path:, payload:, headers:, verb:, retries: MAX_RETRIES, backoff: INITIAL_BACKOFF)
-          adapter.call(
-            path: path, payload: payload, headers: headers, verb: verb
-          )
-        rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError, SocketError, Net::HTTPBadResponse => e
-          Datadog.logger.debug("Failed to send request with #{e} (#{e.message})")
+          response = nil
+
+          begin
+            response = adapter.call(
+              path: path, payload: payload, headers: headers, verb: verb
+            )
+            return response if response.ok?
+
+            if response.code == 429
+              backoff = (response.header(Ext::Transport::HEADER_RATELIMIT_RESET) || 1).to_i
+
+              Datadog.logger.debug do
+                "Received rate limit response, retrying in #{backoff} seconds from X-RateLimit-Reset header"
+              end
+            elsif response.server_error?
+              Datadog.logger.debug { "Received server error response, retrying in #{backoff} seconds" }
+            else
+              return response
+            end
+          rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError, SocketError, Net::HTTPBadResponse => e
+            Datadog.logger.debug { "Failed to send request with #{e} (#{e.message})" }
 
-          if retries.positive?
+            response = ErrorResponse.new(e)
+          end
+
+          if retries.positive? && backoff <= MAX_BACKOFF
             sleep(backoff)
 
             perform_http_call(
-              path: path, payload: payload, headers: headers, verb: verb, retries: retries - 1, backoff: backoff * 2
+              path: path,
+              payload: payload,
+              headers: headers,
+              verb: verb,
+              retries: retries - 1,
+              backoff: backoff * 2
             )
           else
-            Datadog.logger.error("Failed to send request after #{MAX_RETRIES} retries")
-            ErrorResponse.new(e)
+            Datadog.logger.error(
+              "Failed to send request after #{MAX_RETRIES - retries} retries (current backoff value #{backoff})"
+            )
+
+            response
           end
         end
 
@@ -121,6 +149,10 @@ module Datadog
             nil
           end
 
+          def response_size
+            0
+          end
+
           def inspect
             "ErrorResponse error:#{error}"
           end

data/lib/datadog/ci/version.rb

@@ -4,8 +4,8 @@ module Datadog
   module CI
     module VERSION
       MAJOR = 1
-      MINOR = 7
-      PATCH = 0
+      MINOR = 8
+      PATCH = 1
       PRE = nil
       BUILD = nil
       # PRE and BUILD above are modified for dev gems during gem build GHA workflow

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datadog-ci
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.8.1
 platform: ruby
 authors:
 - Datadog, Inc.
 autorequire: 
-bindir: bin
+bindir: exe
 cert_chain: []
-date: 2024-09-25 00:00:00.000000000 Z
+date: 2024-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: datadog
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
@@ -44,7 +44,8 @@ description: |2
     their CI pipelines.
 email:
 - dev@datadoghq.com
-executables: []
+executables:
+- ddcirb
 extensions:
 - ext/datadog_cov/extconf.rb
 extra_rdoc_files: []
@@ -56,9 +57,14 @@ files:
 - LICENSE.BSD3
 - NOTICE
 - README.md
+- exe/ddcirb
 - ext/datadog_cov/datadog_cov.c
 - ext/datadog_cov/extconf.rb
 - lib/datadog/ci.rb
+- lib/datadog/ci/cli/cli.rb
+- lib/datadog/ci/cli/command/base.rb
+- lib/datadog/ci/cli/command/skippable_tests_percentage.rb
+- lib/datadog/ci/cli/command/skippable_tests_percentage_estimate.rb
 - lib/datadog/ci/codeowners/matcher.rb
 - lib/datadog/ci/codeowners/parser.rb
 - lib/datadog/ci/codeowners/rule.rb
@@ -154,6 +160,9 @@ files:
 - lib/datadog/ci/test_optimisation/coverage/transport.rb
 - lib/datadog/ci/test_optimisation/coverage/writer.rb
 - lib/datadog/ci/test_optimisation/skippable.rb
+- lib/datadog/ci/test_optimisation/skippable_percentage/base.rb
+- lib/datadog/ci/test_optimisation/skippable_percentage/calculator.rb
+- lib/datadog/ci/test_optimisation/skippable_percentage/estimator.rb
 - lib/datadog/ci/test_optimisation/telemetry.rb
 - lib/datadog/ci/test_retries/component.rb
 - lib/datadog/ci/test_retries/driver/base.rb
@@ -172,6 +181,7 @@ files:
 - lib/datadog/ci/test_visibility/context.rb
 - lib/datadog/ci/test_visibility/flush.rb
 - lib/datadog/ci/test_visibility/null_component.rb
+- lib/datadog/ci/test_visibility/null_transport.rb
 - lib/datadog/ci/test_visibility/serializers/base.rb
 - lib/datadog/ci/test_visibility/serializers/factories/test_level.rb
 - lib/datadog/ci/test_visibility/serializers/factories/test_suite_level.rb