datadog-ci 1.30.0 → 1.31.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 129bc549b30318587b435c3836517faa058dac0860f07eca408fc5562186e52f
-  data.tar.gz: 4a8c2756b9c846a4b5b225c51e0d1b3910e24c550f60adce4b47f968981a2b8d
+  metadata.gz: de36af96e9eef5f7bffcfe3a3cb8e80ff08461b8f10795bff6a54e53d340a1c6
+  data.tar.gz: 5bdf32ef644ecaac3c57b55f94338359db56e18bea76fec6c40c76fbd1a907bd
 SHA512:
-  metadata.gz: ee1ceadbb2c83d09c780bfe517d52dd4a55b5a4a4dc6802fea1e653fe06b308e7eda9a90a420c1ff855b59b27a7ada8253f319f55664ade19cceb5cab79e3c60
-  data.tar.gz: f297af41a17c77ef5c123ad71a79e92a489218bea1b081d802f414a8496fe7e7ca4da50cde169c290ea3a6c0b189a3987f88f07cdf54bb4c822958527ac26117
+  metadata.gz: 702a1219da6c81acc12d6eca8743de08ff97dd5cdf728ff02556bb282673ac913502f49dd4db02bb4ddd119e8102a9ceacc4a081624a48fe256d3f402efa1887
+  data.tar.gz: 2470ec968a63e625234fe7cb70e9faadf7ce1ccc5a9cf9f04c668c271b693f6b3a3b38f3b5821e27908f299bf58773122b268decbbf3e65596a63cc39509be04

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [Unreleased]
 
+## [1.31.0] - 2026-05-20
+
+### Added
+
+* Add Test Optimization cache readers for Bazel support and DDTest (test parallelization) ([#511][])
+* Add Jenkins custom parent id env var ([#509][])
+
 ## [1.30.0] - 2026-05-12
 
 ### Changed
@@ -634,7 +641,8 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 
 - Ruby versions < 2.7 no longer supported ([#8][])
 
-[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.30.0...main
+[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v1.31.0...main
+[1.31.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.30.0...v1.31.0
 [1.30.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.29.0...v1.30.0
 [1.29.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.28.0...v1.29.0
 [1.28.0]: https://github.com/DataDog/datadog-ci-rb/compare/v1.27.0...v1.28.0
@@ -898,4 +906,6 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 [#499]: https://github.com/DataDog/datadog-ci-rb/issues/499
 [#500]: https://github.com/DataDog/datadog-ci-rb/issues/500
 [#501]: https://github.com/DataDog/datadog-ci-rb/issues/501
-[#506]: https://github.com/DataDog/datadog-ci-rb/issues/506
+[#506]: https://github.com/DataDog/datadog-ci-rb/issues/506
+[#509]: https://github.com/DataDog/datadog-ci-rb/issues/509
+[#511]: https://github.com/DataDog/datadog-ci-rb/issues/511

data/lib/datadog/ci/configuration/components.rb

@@ -17,6 +17,8 @@ require_relative "../remote/library_settings_client"
 require_relative "../test_management/component"
 require_relative "../test_management/null_component"
 require_relative "../test_management/tests_properties"
+require_relative "../test_optimization_cache/component"
+require_relative "../test_optimization_cache/null_component"
 require_relative "../test_impact_analysis/null_component"
 require_relative "../test_impact_analysis/component"
 require_relative "../test_impact_analysis/coverage/transport"
@@ -45,7 +47,8 @@ module Datadog
       # Adds CI behavior to Datadog trace components
       module Components
         attr_reader :test_tracing, :test_impact_analysis, :git_tree_upload_worker, :ci_remote, :test_retries,
-          :test_management, :agentless_logs_submission, :impacted_tests_detection, :test_discovery, :code_coverage
+          :test_management, :agentless_logs_submission, :impacted_tests_detection, :test_discovery, :code_coverage,
+          :test_optimization_cache
 
         def initialize(settings)
           @test_impact_analysis = TestImpactAnalysis::NullComponent.new
@@ -57,6 +60,7 @@ module Datadog
           @impacted_tests_detection = ImpactedTestsDetection::NullComponent.new
           @test_discovery = TestDiscovery::NullComponent.new
           @code_coverage = CodeCoverage::NullComponent.new
+          @test_optimization_cache = TestOptimizationCache::NullComponent.new
 
           # Activate CI mode if enabled
           if settings.ci.enabled
@@ -69,6 +73,7 @@ module Datadog
         def shutdown!(replacement = nil)
           super
 
+          @test_optimization_cache&.shutdown!
           @test_tracing&.shutdown!
           @test_impact_analysis&.shutdown!
           @agentless_logs_submission&.shutdown!
@@ -113,6 +118,13 @@ module Datadog
           )
           @test_discovery.disable_features_for_test_discovery!(settings)
 
+          @test_optimization_cache = TestOptimizationCache::Component.new(
+            manifest_file: settings.ci.test_optimization_cache_manifest_file,
+            runfiles_dir: settings.ci.test_optimization_cache_runfiles_dir,
+            runfiles_manifest_file: settings.ci.test_optimization_cache_runfiles_manifest_file,
+            test_srcdir: settings.ci.test_optimization_cache_test_srcdir
+          )
+
           # Configure Datadog::Tracing module
 
           # No need not use 128-bit trace ids for test visibility,

data/lib/datadog/ci/configuration/settings.rb

@@ -2,6 +2,7 @@
 
 require_relative "../contrib/instrumentation"
 require_relative "../ext/settings"
+require_relative "../ext/test_optimization_cache"
 require_relative "../utils/bundle"
 require_relative "../utils/parsing"
 
@@ -180,6 +181,26 @@ module Datadog
                 o.env CI::Ext::Settings::ENV_TEST_DISCOVERY_OUTPUT_PATH
               end
 
+              option :test_optimization_cache_manifest_file do |o|
+                o.type :string, nilable: true
+                o.env CI::Ext::TestOptimizationCache::ENV_MANIFEST_FILE
+              end
+
+              option :test_optimization_cache_runfiles_dir do |o|
+                o.type :string, nilable: true
+                o.env CI::Ext::TestOptimizationCache::ENV_RUNFILES_DIR
+              end
+
+              option :test_optimization_cache_runfiles_manifest_file do |o|
+                o.type :string, nilable: true
+                o.env CI::Ext::TestOptimizationCache::ENV_RUNFILES_MANIFEST_FILE
+              end
+
+              option :test_optimization_cache_test_srcdir do |o|
+                o.type :string, nilable: true
+                o.env CI::Ext::TestOptimizationCache::ENV_TEST_SRCDIR
+              end
+
               option :tia_static_dependencies_tracking_enabled do |o|
                 o.type :bool
                 o.env CI::Ext::Settings::ENV_TIA_STATIC_DEPENDENCIES_TRACKING_ENABLED

data/lib/datadog/ci/ext/environment/providers/jenkins.rb

@@ -75,7 +75,8 @@ module Datadog
 
             def ci_env_vars
               {
-                "DD_CUSTOM_TRACE_ID" => env["DD_CUSTOM_TRACE_ID"]
+                "DD_CUSTOM_TRACE_ID" => env["DD_CUSTOM_TRACE_ID"],
+                "DD_CUSTOM_PARENT_ID" => env["DD_CUSTOM_PARENT_ID"]
               }.to_json
             end
           end

data/lib/datadog/ci/ext/test_discovery.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
+require_relative "test_optimization_cache"
+
 module Datadog
   module CI
     module Ext
       # Defines constants for test discovery mode
       module TestDiscovery
         # Default output path for test discovery mode
-        DEFAULT_OUTPUT_PATH = "./#{DDTest::PLAN_FOLDER}/test_discovery/tests.json"
+        DEFAULT_OUTPUT_PATH = "./#{TestOptimizationCache::PLAN_FOLDER}/test_discovery/tests.json"
 
         # Maximum buffer size before writing to file
         MAX_BUFFER_SIZE = 10_000

data/lib/datadog/ci/ext/test_optimization_cache.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Ext
+      # Constants for Test Optimization cache generated by ddtest or Bazel.
+      module TestOptimizationCache
+        ENV_MANIFEST_FILE = "TEST_OPTIMIZATION_MANIFEST_FILE"
+        ENV_RUNFILES_DIR = "RUNFILES_DIR"
+        ENV_RUNFILES_MANIFEST_FILE = "RUNFILES_MANIFEST_FILE"
+        ENV_TEST_SRCDIR = "TEST_SRCDIR"
+
+        PLAN_FOLDER = ".testoptimization"
+        MANIFEST_FILE_NAME = "manifest.txt"
+        SUPPORTED_MANIFEST_VERSION = "1"
+
+        CACHE_FOLDER_NAME = "cache"
+        HTTP_CACHE_FOLDER_NAME = "http"
+
+        TESTOPTIMIZATION_CACHE_PATH = "#{PLAN_FOLDER}/#{CACHE_FOLDER_NAME}"
+        TESTOPTIMIZATION_HTTP_CACHE_PATH = "#{TESTOPTIMIZATION_CACHE_PATH}/#{HTTP_CACHE_FOLDER_NAME}"
+
+        SETTINGS_FILE_NAME = "settings.json"
+        KNOWN_TESTS_FILE_NAME = "known_tests.json"
+        TEST_MANAGEMENT_FILE_NAME = "test_management.json"
+        LEGACY_TEST_MANAGEMENT_TESTS_FILE_NAME = "test_management_tests.json"
+        SKIPPABLE_TESTS_FILE_NAME = "skippable_tests.json"
+      end
+    end
+  end
+end

data/lib/datadog/ci/git/tree_uploader.rb

@@ -10,7 +10,6 @@ require_relative "packfiles"
 
 require_relative "../ext/telemetry"
 require_relative "../utils/telemetry"
-require_relative "../utils/test_run"
 
 module Datadog
   module CI
@@ -34,8 +33,8 @@ module Datadog
             return
           end
 
-          if Utils::TestRun.test_optimization_data_cached?
-            Datadog.logger.debug("DDTest cache found, git upload already done by DDTest tool, skipping git upload")
+          if test_optimization_cache.cache_available?
+            Datadog.logger.debug("Test Optimization cache found, git upload already done, skipping git upload")
             return
           end
 
@@ -113,6 +112,10 @@ module Datadog
         def test_tracing_component
           Datadog.send(:components).test_tracing
         end
+
+        def test_optimization_cache
+          Datadog.send(:components).test_optimization_cache
+        end
       end
     end
   end

data/lib/datadog/ci/remote/component.rb

@@ -56,15 +56,15 @@ module Datadog
         end
 
         def restore_state_from_datadog_test_runner
-          Datadog.logger.debug { "Restoring library configuration from DDTest cache" }
+          Datadog.logger.debug { "Restoring library configuration from Test Optimization cache" }
 
-          settings = load_json(Ext::DDTest::SETTINGS_FILE_NAME)
+          settings = load_cached_settings
           if settings.nil?
             Datadog.logger.debug { "Restoring library configuration failed, will request again" }
             return false
           end
 
-          Datadog.logger.debug { "Restored library configuration from DDTest: #{settings}" }
+          Datadog.logger.debug { "Restored library configuration from Test Optimization: #{settings}" }
           @library_configuration = LibrarySettings.from_json(settings)
           true
         end
@@ -75,7 +75,7 @@ module Datadog
           # In test discovery mode, skip backend fetching and use default settings (everything is disabled)
           return @library_configuration = LibrarySettings.from_http_response(nil) if @test_discovery_enabled
 
-          # skip backend request if library configuration was loaded by a different process and stored on disk
+          # skip backend request if library configuration was loaded from external cache or stored on disk
           library_configuration_loaded = load_component_state
           return @library_configuration if library_configuration_loaded
 

data/lib/datadog/ci/remote/library_settings.rb

@@ -19,7 +19,13 @@ module Datadog
         end
 
         def self.from_json(json)
-          new(nil, json)
+          new(nil, normalize_json(json))
+        end
+
+        def self.normalize_json(json)
+          return nil if json.nil?
+
+          json.dig(*Ext::Transport::DD_API_SETTINGS_RESPONSE_DIG_KEYS) || json
         end
 
         def ok?

data/lib/datadog/ci/test_impact_analysis/component.rb

@@ -6,7 +6,6 @@ require "datadog/core/telemetry/logging"
 
 require_relative "../ext/test"
 require_relative "../ext/telemetry"
-require_relative "../ext/dd_test"
 
 require_relative "../git/local_repository"
 
@@ -102,11 +101,9 @@ module Datadog
             populate_static_dependencies_map!
           end
 
-          # Load component state first, and if successful, skip fetching skippable tests
-          # Also try to restore from DDTest cache if available
+          # Load external cache or component state first, and if successful, skip fetching skippable tests
           if skipping_tests?
             return if load_component_state
-            return if restore_state_from_datadog_test_runner
 
             fetch_skippable_tests(test_session)
             store_component_state if test_session.distributed
@@ -333,22 +330,17 @@ module Datadog
         end
 
         def restore_state_from_datadog_test_runner
-          Datadog.logger.debug { "Restoring skippable tests from DDTest cache" }
+          Datadog.logger.debug { "Restoring skippable tests from Test Optimization cache" }
 
-          skippable_tests_data = load_json(Ext::DDTest::SKIPPABLE_TESTS_FILE_NAME)
+          skippable_tests_data = load_cached_skippable_tests
           if skippable_tests_data.nil?
             Datadog.logger.debug { "Restoring skippable tests failed, will request again" }
             return false
           end
 
-          Datadog.logger.debug { "Restored skippable tests from DDTest: #{skippable_tests_data}" }
+          Datadog.logger.debug { "Restored skippable tests from Test Optimization: #{skippable_tests_data}" }
 
-          transformed_data = transform_test_runner_data(skippable_tests_data)
-
-          Datadog.logger.debug { "Skippable tests after transformation: #{transformed_data}" }
-
-          # Use the Skippable::Response class to parse the transformed data
-          skippable_response = Skippable::Response.from_json(transformed_data)
+          skippable_response = Skippable::Response.from_json(skippable_tests_data)
 
           @mutex.synchronize do
             @correlation_id = skippable_response.correlation_id
@@ -363,58 +355,6 @@ module Datadog
 
         private
 
-        # Transforms Test Runner skippable tests data format to the format expected by Skippable::Response
-        #
-        # Test Runner format:
-        # {
-        #   "correlationId": "abc123",
-        #   "skippableTests": {
-        #     "suite_name": {
-        #       "test_name": [{"suite": "suite_name", "name": "test_name", "parameters": "{...}", "configurations": {}}]
-        #     }
-        #   }
-        # }
-        #
-        # Expected format:
-        # {
-        #   "meta": {"correlation_id": "abc123"},
-        #   "data": [{"type": "test", "attributes": {"suite": "suite_name", "name": "test_name", "parameters": "{...}"}}]
-        # }
-        def transform_test_runner_data(skippable_tests_data)
-          skippable_tests = skippable_tests_data.fetch("skippableTests", {}) || {}
-
-          # Pre-calculate array size for better memory allocation
-          total_test_configs = skippable_tests.sum do |_, tests_hash|
-            tests_hash.sum { |_, test_configs| test_configs.size }
-          end
-
-          data_array = Array.new(total_test_configs)
-          index = 0
-
-          skippable_tests.each_value do |tests_hash|
-            tests_hash.each_value do |test_configs|
-              test_configs.each do |test_config|
-                data_array[index] = {
-                  "type" => Ext::Test::ITR_TEST_SKIPPING_MODE,
-                  "attributes" => {
-                    "suite" => test_config["suite"],
-                    "name" => test_config["name"],
-                    "parameters" => test_config["parameters"]
-                  }
-                }
-                index += 1
-              end
-            end
-          end
-
-          {
-            "meta" => {
-              "correlation_id" => skippable_tests_data["correlationId"]
-            },
-            "data" => data_array
-          }
-        end
-
         def write(event)
           # skip sending events if writer is not configured
           @coverage_writer&.write(event)

data/lib/datadog/ci/test_management/component.rb

@@ -2,7 +2,6 @@
 
 require_relative "../ext/telemetry"
 require_relative "../ext/test"
-require_relative "../ext/dd_test"
 require_relative "../utils/stateful"
 require_relative "../utils/telemetry"
 require_relative "../utils/test_run"
@@ -36,7 +35,6 @@ module Datadog
 
           test_session.set_tag(Ext::Test::TAG_TEST_MANAGEMENT_ENABLED, "true")
 
-          return if restore_state_from_datadog_test_runner
           return if load_component_state
 
           @tests_properties = @tests_properties_client.fetch(test_session)
@@ -85,25 +83,17 @@ module Datadog
         end
 
         def restore_state_from_datadog_test_runner
-          Datadog.logger.debug { "Restoring test management tests from DDTest cache" }
+          Datadog.logger.debug { "Restoring test management tests from Test Optimization cache" }
 
-          test_management_data = load_json(Ext::DDTest::TEST_MANAGEMENT_TESTS_FILE_NAME)
+          test_management_data = load_cached_test_management
           if test_management_data.nil?
             Datadog.logger.debug { "Restoring test management tests failed, will request again" }
             return false
           end
 
-          Datadog.logger.debug { "Restored test management tests from DDTest: #{test_management_data}" }
+          Datadog.logger.debug { "Restored test management tests from Test Optimization: #{test_management_data}" }
 
-          # Use the TestsProperties::Response class method to parse the JSON data
-          # Wrap the data in the expected backend API format
-          wrapped_data = {
-            "data" => {
-              "attributes" => test_management_data
-            }
-          }
-
-          tests_properties_response = TestsProperties::Response.from_json(wrapped_data)
+          tests_properties_response = TestsProperties::Response.from_json(test_management_data)
           @tests_properties = tests_properties_response.tests
 
           Datadog.logger.debug { "Found [#{@tests_properties.size}] test management tests from context" }

data/lib/datadog/ci/test_optimization_cache/component.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require_relative "../ext/test_optimization_cache"
+require_relative "locator"
+require_relative "readers/legacy"
+require_relative "readers/missing"
+require_relative "readers/v1"
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      class Component
+        READER_BY_MANIFEST_VERSION = {
+          Ext::TestOptimizationCache::SUPPORTED_MANIFEST_VERSION => Readers::V1
+        }.freeze
+
+        def initialize(manifest_file:, runfiles_dir:, runfiles_manifest_file:, test_srcdir:)
+          @locator = Locator.new(
+            manifest_file: manifest_file,
+            runfiles_dir: runfiles_dir,
+            runfiles_manifest_file: runfiles_manifest_file,
+            test_srcdir: test_srcdir
+          )
+          @reader = build_reader
+        end
+
+        def cache_available?
+          @reader.available?
+        end
+
+        def load_settings
+          @reader.load_settings
+        end
+
+        def load_known_tests
+          @reader.load_known_tests
+        end
+
+        def load_test_management
+          @reader.load_test_management
+        end
+
+        def load_skippable_tests
+          @reader.load_skippable_tests
+        end
+
+        def shutdown!
+        end
+
+        private
+
+        def build_reader
+          manifest_path = @locator.manifest_path
+
+          if manifest_path
+            version = @locator.manifest_version(manifest_path)
+            reader_class = READER_BY_MANIFEST_VERSION[version] if version
+            if reader_class
+              test_optimization_path = File.dirname(manifest_path)
+              reader = reader_class.new(test_optimization_path)
+              return reader if reader.available?
+
+              Datadog.logger.debug do
+                "Test Optimization cache settings file not found under #{test_optimization_path}"
+              end
+              return Readers::Missing.new
+            end
+
+            Datadog.logger.debug do
+              "Unsupported Test Optimization cache manifest version #{version.inspect} at #{manifest_path}"
+            end
+            return Readers::Missing.new
+          end
+
+          return Readers::Legacy.new if Dir.exist?(Ext::TestOptimizationCache::TESTOPTIMIZATION_CACHE_PATH)
+
+          Readers::Missing.new
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimization_cache/locator.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require_relative "../ext/test_optimization_cache"
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      class Locator
+        RUNFILES_MANIFEST_SEPARATOR = " "
+
+        def initialize(manifest_file:, runfiles_dir:, runfiles_manifest_file:, test_srcdir:)
+          @manifest_file = manifest_file
+          @runfiles_dir = runfiles_dir
+          @runfiles_manifest_file = runfiles_manifest_file
+          @test_srcdir = test_srcdir
+          @manifest_versions = {}
+        end
+
+        def manifest_path
+          return @manifest_path if defined?(@manifest_path)
+
+          local_manifest = File.join(Ext::TestOptimizationCache::PLAN_FOLDER, Ext::TestOptimizationCache::MANIFEST_FILE_NAME)
+          if File.exist?(local_manifest)
+            @manifest_path = local_manifest
+            return @manifest_path
+          end
+
+          manifest_file = @manifest_file
+          if manifest_file && !manifest_file.empty?
+            env_path = resolve_bazel_runfile_path(manifest_file)
+            if File.exist?(env_path)
+              @manifest_path = env_path
+              return @manifest_path
+            end
+          end
+
+          @manifest_path = nil
+        end
+
+        def manifest_version(manifest_path)
+          return @manifest_versions[manifest_path] if @manifest_versions.key?(manifest_path)
+
+          @manifest_versions[manifest_path] = File.read(manifest_path).delete_prefix("\uFEFF").strip
+        rescue => e
+          Datadog.logger.debug { "Failed to read Test Optimization cache manifest #{manifest_path}: #{e.message}" }
+          @manifest_versions[manifest_path] = nil
+        end
+
+        def resolve_bazel_runfile_path(path)
+          return path if File.exist?(path)
+
+          runfiles_dir = @runfiles_dir
+          if runfiles_dir && !runfiles_dir.empty?
+            candidate = File.join(runfiles_dir, path)
+            return candidate if File.exist?(candidate)
+          end
+
+          manifest_candidate = resolve_bazel_runfile_path_from_manifest(path)
+          return manifest_candidate if manifest_candidate
+
+          test_srcdir = @test_srcdir
+          if test_srcdir && !test_srcdir.empty?
+            candidate = File.join(test_srcdir, path)
+            return candidate if File.exist?(candidate)
+          end
+
+          path
+        end
+
+        # Bazel can provide runfiles through a manifest file instead of a directory tree.
+        # Each line maps a logical runfile path to the actual path on disk.
+        def resolve_bazel_runfile_path_from_manifest(path)
+          runfiles_manifest = @runfiles_manifest_file
+          return nil if runfiles_manifest.nil? || runfiles_manifest.empty?
+          return nil unless File.exist?(runfiles_manifest)
+
+          File.foreach(runfiles_manifest) do |line|
+            separator_index = line.index(RUNFILES_MANIFEST_SEPARATOR)
+            next unless separator_index&.positive?
+            next unless line[0...separator_index] == path
+
+            resolved_path = line[(separator_index + 1)..]
+            return resolved_path.strip if resolved_path
+          end
+
+          nil
+        rescue => e
+          Datadog.logger.debug do
+            "Failed to resolve Test Optimization cache manifest from #{runfiles_manifest}: #{e.message}"
+          end
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimization_cache/null_component.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      class NullComponent
+        def cache_available?
+          false
+        end
+
+        def load_settings
+          nil
+        end
+
+        def load_known_tests
+          nil
+        end
+
+        def load_test_management
+          nil
+        end
+
+        def load_skippable_tests
+          nil
+        end
+
+        def shutdown!
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimization_cache/readers/base.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "../../utils/json"
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      module Readers
+        class Base
+          def available?
+            true
+          end
+
+          def load_settings
+            raise NotImplementedError, "#{self.class} must implement #load_settings"
+          end
+
+          def load_known_tests
+            raise NotImplementedError, "#{self.class} must implement #load_known_tests"
+          end
+
+          def load_test_management
+            raise NotImplementedError, "#{self.class} must implement #load_test_management"
+          end
+
+          def load_skippable_tests
+            raise NotImplementedError, "#{self.class} must implement #load_skippable_tests"
+          end
+
+          private
+
+          def read_json_file(file_path)
+            Utils::Json.read_file(file_path)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimization_cache/readers/legacy.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "../../ext/test_optimization_cache"
+require_relative "base"
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      module Readers
+        class Legacy < Base
+          def load_settings
+            load_legacy_json(Ext::TestOptimizationCache::SETTINGS_FILE_NAME)
+          end
+
+          def load_known_tests
+            payload = load_legacy_json(Ext::TestOptimizationCache::KNOWN_TESTS_FILE_NAME)
+            backend_response(payload) if payload
+          end
+
+          def load_test_management
+            payload = load_legacy_json(Ext::TestOptimizationCache::LEGACY_TEST_MANAGEMENT_TESTS_FILE_NAME)
+            backend_response(payload) if payload
+          end
+
+          def load_skippable_tests
+            payload = load_legacy_json(Ext::TestOptimizationCache::SKIPPABLE_TESTS_FILE_NAME)
+            skippable_tests_response(payload) if payload
+          end
+
+          private
+
+          def load_legacy_json(file_name)
+            read_json_file(File.join(Ext::TestOptimizationCache::TESTOPTIMIZATION_CACHE_PATH, file_name))
+          end
+
+          def backend_response(payload)
+            {
+              "data" => {
+                "attributes" => payload
+              }
+            }
+          end
+
+          def skippable_tests_response(payload)
+            skippable_tests = payload.fetch("skippableTests", {}) || {}
+
+            data = skippable_tests.each_value.flat_map do |tests_hash|
+              tests_hash.each_value.flat_map do |test_configs|
+                test_configs.map do |test_config|
+                  {
+                    "type" => Ext::Test::ITR_TEST_SKIPPING_MODE,
+                    "attributes" => {
+                      "suite" => test_config["suite"],
+                      "name" => test_config["name"],
+                      "parameters" => test_config["parameters"]
+                    }
+                  }
+                end
+              end
+            end
+
+            {
+              "meta" => {
+                "correlation_id" => payload["correlationId"]
+              },
+              "data" => data
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimization_cache/readers/missing.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      module Readers
+        class Missing < Base
+          def available?
+            false
+          end
+
+          def load_settings
+            nil
+          end
+
+          def load_known_tests
+            nil
+          end
+
+          def load_test_management
+            nil
+          end
+
+          def load_skippable_tests
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimization_cache/readers/v1.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test_optimization_cache"
+require_relative "base"
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      module Readers
+        class V1 < Base
+          def initialize(test_optimization_path)
+            @test_optimization_path = test_optimization_path
+          end
+
+          def available?
+            File.exist?(settings_file_path)
+          end
+
+          def load_settings
+            load_http_json(Ext::TestOptimizationCache::SETTINGS_FILE_NAME)
+          end
+
+          def load_known_tests
+            load_http_json(Ext::TestOptimizationCache::KNOWN_TESTS_FILE_NAME)
+          end
+
+          def load_test_management
+            load_http_json(Ext::TestOptimizationCache::TEST_MANAGEMENT_FILE_NAME)
+          end
+
+          def load_skippable_tests
+            load_http_json(Ext::TestOptimizationCache::SKIPPABLE_TESTS_FILE_NAME)
+          end
+
+          private
+
+          def load_http_json(file_name)
+            read_json_file(File.join(http_cache_path, file_name))
+          end
+
+          def settings_file_path
+            File.join(http_cache_path, Ext::TestOptimizationCache::SETTINGS_FILE_NAME)
+          end
+
+          def http_cache_path
+            File.join(
+              @test_optimization_path,
+              Ext::TestOptimizationCache::CACHE_FOLDER_NAME,
+              Ext::TestOptimizationCache::HTTP_CACHE_FOLDER_NAME
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_tracing/component.rb

@@ -225,28 +225,21 @@ module Datadog
         end
 
         def restore_state_from_datadog_test_runner
-          Datadog.logger.debug { "Restoring known tests from DDTest cache" }
+          Datadog.logger.debug { "Restoring known tests from Test Optimization cache" }
 
-          known_tests_data = load_json(Ext::DDTest::KNOWN_TESTS_FILE_NAME)
+          known_tests_data = load_cached_known_tests
           if known_tests_data.nil?
             Datadog.logger.debug { "Restoring known tests failed, will request again" }
             return false
           end
 
-          Datadog.logger.debug { "Restored known tests from DDTest: #{known_tests_data}" }
-
-          # Use the KnownTests class method to parse the JSON data
-          known_tests_data = {
-            "data" => {
-              "attributes" => known_tests_data
-            }
-          }
+          Datadog.logger.debug { "Restored known tests from Test Optimization: #{known_tests_data}" }
 
           @known_tests = KnownTests::Response.from_json(known_tests_data).tests
           @known_tests_enabled = !@known_tests.empty?
 
           unless @known_tests_enabled
-            Datadog.logger.debug("Empty set of known tests from the DDTest cache file")
+            Datadog.logger.debug("Empty set of known tests from the Test Optimization cache file")
           end
 
           Datadog.logger.debug { "Found [#{@known_tests.size}] known tests from context" }

data/lib/datadog/ci/utils/json.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Datadog
+  module CI
+    module Utils
+      module Json
+        def self.read_file(file_path)
+          unless File.exist?(file_path)
+            Datadog.logger.debug { "JSON file not found: #{file_path}" }
+            return nil
+          end
+
+          JSON.parse(File.read(file_path))
+        rescue JSON::ParserError, SystemCallError => e
+          Datadog.logger.debug { "Failed to load JSON file #{file_path}: #{e.message}" }
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/utils/stateful.rb

@@ -1,9 +1,6 @@
 # frozen_string_literal: true
 
-require "json"
 require_relative "file_storage"
-require_relative "test_run"
-require_relative "../ext/dd_test"
 
 module Datadog
   module CI
@@ -22,9 +19,8 @@ module Datadog
 
         # Load component state
         def load_component_state
-          # Check for DDTest cache first
-          if TestRun.test_optimization_data_cached?
-            Datadog.logger.debug { "DDTest cache found" }
+          if test_optimization_cache.cache_available?
+            Datadog.logger.debug { "Test Optimization cache found" }
             return true if restore_state_from_datadog_test_runner
           end
 
@@ -60,22 +56,24 @@ module Datadog
           false
         end
 
-        def load_json(file_name)
-          file_path = File.join(Ext::DDTest::TESTOPTIMIZATION_CACHE_PATH, file_name)
+        def load_cached_settings
+          test_optimization_cache.load_settings
+        end
 
-          unless File.exist?(file_path)
-            Datadog.logger.debug { "JSON file not found: #{file_path}" }
-            return nil
-          end
+        def load_cached_known_tests
+          test_optimization_cache.load_known_tests
+        end
+
+        def load_cached_test_management
+          test_optimization_cache.load_test_management
+        end
+
+        def load_cached_skippable_tests
+          test_optimization_cache.load_skippable_tests
+        end
 
-          content = File.read(file_path)
-          JSON.parse(content)
-        rescue JSON::ParserError => e
-          Datadog.logger.debug { "Failed to parse JSON file #{file_path}: #{e.message}" }
-          nil
-        rescue => e
-          Datadog.logger.debug { "Failed to load JSON file #{file_path}: #{e.message}" }
-          nil
+        def test_optimization_cache
+          Datadog.send(:components).test_optimization_cache
         end
       end
     end

data/lib/datadog/ci/utils/test_run.rb

@@ -2,8 +2,6 @@
 
 require "etc"
 
-require_relative "../ext/dd_test"
-
 module Datadog
   module CI
     module Utils
@@ -44,10 +42,6 @@ module Datadog
 
           @virtual_cpu_count = ::Etc.nprocessors
         end
-
-        def self.test_optimization_data_cached?
-          Dir.exist?(Ext::DDTest::TESTOPTIMIZATION_CACHE_PATH)
-        end
       end
     end
   end

data/lib/datadog/ci/version.rb

@@ -4,7 +4,7 @@ module Datadog
   module CI
     module VERSION
       MAJOR = 1
-      MINOR = 30
+      MINOR = 31
       PATCH = 0
       PRE = nil
       BUILD = nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datadog-ci
 version: !ruby/object:Gem::Version
-  version: 1.30.0
+  version: 1.31.0
 platform: ruby
 authors:
 - Datadog, Inc.
@@ -178,7 +178,6 @@ files:
 - lib/datadog/ci/contrib/simplecov/report_uploader.rb
 - lib/datadog/ci/contrib/simplecov/result_extractor.rb
 - lib/datadog/ci/ext/app_types.rb
-- lib/datadog/ci/ext/dd_test.rb
 - lib/datadog/ci/ext/environment.rb
 - lib/datadog/ci/ext/environment/configuration_discrepancy_checker.rb
 - lib/datadog/ci/ext/environment/extractor.rb
@@ -207,6 +206,7 @@ files:
 - lib/datadog/ci/ext/telemetry.rb
 - lib/datadog/ci/ext/test.rb
 - lib/datadog/ci/ext/test_discovery.rb
+- lib/datadog/ci/ext/test_optimization_cache.rb
 - lib/datadog/ci/ext/transport.rb
 - lib/datadog/ci/git/base_branch_sha_detection/base.rb
 - lib/datadog/ci/git/base_branch_sha_detection/branch_metric.rb
@@ -257,6 +257,13 @@ files:
 - lib/datadog/ci/test_management/null_component.rb
 - lib/datadog/ci/test_management/tests_properties.rb
 - lib/datadog/ci/test_module.rb
+- lib/datadog/ci/test_optimization_cache/component.rb
+- lib/datadog/ci/test_optimization_cache/locator.rb
+- lib/datadog/ci/test_optimization_cache/null_component.rb
+- lib/datadog/ci/test_optimization_cache/readers/base.rb
+- lib/datadog/ci/test_optimization_cache/readers/legacy.rb
+- lib/datadog/ci/test_optimization_cache/readers/missing.rb
+- lib/datadog/ci/test_optimization_cache/readers/v1.rb
 - lib/datadog/ci/test_retries/component.rb
 - lib/datadog/ci/test_retries/driver/base.rb
 - lib/datadog/ci/test_retries/driver/no_retry.rb
@@ -307,6 +314,7 @@ files:
 - lib/datadog/ci/utils/configuration.rb
 - lib/datadog/ci/utils/file_storage.rb
 - lib/datadog/ci/utils/git.rb
+- lib/datadog/ci/utils/json.rb
 - lib/datadog/ci/utils/parsing.rb
 - lib/datadog/ci/utils/rum.rb
 - lib/datadog/ci/utils/stateful.rb

data/lib/datadog/ci/ext/dd_test.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Datadog
-  module CI
-    module Ext
-      # Constants for integration with DDTest tool: https://github.com/DataDog/ddtest
-      module DDTest
-        PLAN_FOLDER = ".testoptimization"
-        TESTOPTIMIZATION_CACHE_PATH = "#{PLAN_FOLDER}/cache"
-
-        SETTINGS_FILE_NAME = "settings.json"
-        KNOWN_TESTS_FILE_NAME = "known_tests.json"
-        TEST_MANAGEMENT_TESTS_FILE_NAME = "test_management_tests.json"
-        SKIPPABLE_TESTS_FILE_NAME = "skippable_tests.json"
-      end
-    end
-  end
-end