datadog-ci 1.30.0 → 1.32.0

data/lib/datadog/ci/test_optimization_cache/component.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require_relative "../ext/test_optimization_cache"
+require_relative "locator"
+require_relative "readers/legacy"
+require_relative "readers/missing"
+require_relative "readers/v1"
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      class Component
+        READER_BY_MANIFEST_VERSION = {
+          Ext::TestOptimizationCache::SUPPORTED_MANIFEST_VERSION => Readers::V1
+        }.freeze
+
+        def initialize(manifest_file:, runfiles_dir:, runfiles_manifest_file:, test_srcdir:)
+          @locator = Locator.new(
+            manifest_file: manifest_file,
+            runfiles_dir: runfiles_dir,
+            runfiles_manifest_file: runfiles_manifest_file,
+            test_srcdir: test_srcdir
+          )
+          @reader = build_reader
+        end
+
+        def cache_available?
+          @reader.available?
+        end
+
+        def load_settings
+          @reader.load_settings
+        end
+
+        def load_known_tests
+          @reader.load_known_tests
+        end
+
+        def load_test_management
+          @reader.load_test_management
+        end
+
+        def load_skippable_tests
+          @reader.load_skippable_tests
+        end
+
+        def shutdown!
+        end
+
+        private
+
+        def build_reader
+          manifest_path = @locator.manifest_path
+
+          if manifest_path
+            version = @locator.manifest_version(manifest_path)
+            reader_class = READER_BY_MANIFEST_VERSION[version] if version
+            if reader_class
+              test_optimization_path = File.dirname(manifest_path)
+              reader = reader_class.new(test_optimization_path)
+              return reader if reader.available?
+
+              Datadog.logger.debug do
+                "Test Optimization cache settings file not found under #{test_optimization_path}"
+              end
+              return Readers::Missing.new
+            end
+
+            Datadog.logger.debug do
+              "Unsupported Test Optimization cache manifest version #{version.inspect} at #{manifest_path}"
+            end
+            return Readers::Missing.new
+          end
+
+          return Readers::Legacy.new if Dir.exist?(Ext::TestOptimizationCache::TESTOPTIMIZATION_CACHE_PATH)
+
+          Readers::Missing.new
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimization_cache/locator.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require_relative "../ext/test_optimization_cache"
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      class Locator
+        RUNFILES_MANIFEST_SEPARATOR = " "
+
+        def initialize(manifest_file:, runfiles_dir:, runfiles_manifest_file:, test_srcdir:)
+          @manifest_file = manifest_file
+          @runfiles_dir = runfiles_dir
+          @runfiles_manifest_file = runfiles_manifest_file
+          @test_srcdir = test_srcdir
+          @manifest_versions = {}
+        end
+
+        def manifest_path
+          return @manifest_path if defined?(@manifest_path)
+
+          local_manifest = File.join(Ext::TestOptimizationCache::PLAN_FOLDER, Ext::TestOptimizationCache::MANIFEST_FILE_NAME)
+          if File.exist?(local_manifest)
+            @manifest_path = local_manifest
+            return @manifest_path
+          end
+
+          manifest_file = @manifest_file
+          if manifest_file && !manifest_file.empty?
+            env_path = resolve_bazel_runfile_path(manifest_file)
+            if File.exist?(env_path)
+              @manifest_path = env_path
+              return @manifest_path
+            end
+          end
+
+          @manifest_path = nil
+        end
+
+        def manifest_version(manifest_path)
+          return @manifest_versions[manifest_path] if @manifest_versions.key?(manifest_path)
+
+          @manifest_versions[manifest_path] = File.read(manifest_path).delete_prefix("\uFEFF").strip
+        rescue => e
+          Datadog.logger.debug { "Failed to read Test Optimization cache manifest #{manifest_path}: #{e.message}" }
+          @manifest_versions[manifest_path] = nil
+        end
+
+        def resolve_bazel_runfile_path(path)
+          return path if File.exist?(path)
+
+          runfiles_dir = @runfiles_dir
+          if runfiles_dir && !runfiles_dir.empty?
+            candidate = File.join(runfiles_dir, path)
+            return candidate if File.exist?(candidate)
+          end
+
+          manifest_candidate = resolve_bazel_runfile_path_from_manifest(path)
+          return manifest_candidate if manifest_candidate
+
+          test_srcdir = @test_srcdir
+          if test_srcdir && !test_srcdir.empty?
+            candidate = File.join(test_srcdir, path)
+            return candidate if File.exist?(candidate)
+          end
+
+          path
+        end
+
+        # Bazel can provide runfiles through a manifest file instead of a directory tree.
+        # Each line maps a logical runfile path to the actual path on disk.
+        def resolve_bazel_runfile_path_from_manifest(path)
+          runfiles_manifest = @runfiles_manifest_file
+          return nil if runfiles_manifest.nil? || runfiles_manifest.empty?
+          return nil unless File.exist?(runfiles_manifest)
+
+          File.foreach(runfiles_manifest) do |line|
+            separator_index = line.index(RUNFILES_MANIFEST_SEPARATOR)
+            next unless separator_index&.positive?
+            next unless line[0...separator_index] == path
+
+            resolved_path = line[(separator_index + 1)..]
+            return resolved_path.strip if resolved_path
+          end
+
+          nil
+        rescue => e
+          Datadog.logger.debug do
+            "Failed to resolve Test Optimization cache manifest from #{runfiles_manifest}: #{e.message}"
+          end
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimization_cache/null_component.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      class NullComponent
+        def cache_available?
+          false
+        end
+
+        def load_settings
+          nil
+        end
+
+        def load_known_tests
+          nil
+        end
+
+        def load_test_management
+          nil
+        end
+
+        def load_skippable_tests
+          nil
+        end
+
+        def shutdown!
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimization_cache/readers/base.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "../../utils/json"
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      module Readers
+        class Base
+          def available?
+            true
+          end
+
+          def load_settings
+            raise NotImplementedError, "#{self.class} must implement #load_settings"
+          end
+
+          def load_known_tests
+            raise NotImplementedError, "#{self.class} must implement #load_known_tests"
+          end
+
+          def load_test_management
+            raise NotImplementedError, "#{self.class} must implement #load_test_management"
+          end
+
+          def load_skippable_tests
+            raise NotImplementedError, "#{self.class} must implement #load_skippable_tests"
+          end
+
+          private
+
+          def read_json_file(file_path)
+            Utils::Json.read_file(file_path)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimization_cache/readers/legacy.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test"
+require_relative "../../ext/test_optimization_cache"
+require_relative "base"
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      module Readers
+        class Legacy < Base
+          def load_settings
+            load_legacy_json(Ext::TestOptimizationCache::SETTINGS_FILE_NAME)
+          end
+
+          def load_known_tests
+            payload = load_legacy_json(Ext::TestOptimizationCache::KNOWN_TESTS_FILE_NAME)
+            backend_response(payload) if payload
+          end
+
+          def load_test_management
+            payload = load_legacy_json(Ext::TestOptimizationCache::LEGACY_TEST_MANAGEMENT_TESTS_FILE_NAME)
+            backend_response(payload) if payload
+          end
+
+          def load_skippable_tests
+            payload = load_legacy_json(Ext::TestOptimizationCache::SKIPPABLE_TESTS_FILE_NAME)
+            skippable_tests_response(payload) if payload
+          end
+
+          private
+
+          def load_legacy_json(file_name)
+            read_json_file(File.join(Ext::TestOptimizationCache::TESTOPTIMIZATION_CACHE_PATH, file_name))
+          end
+
+          def backend_response(payload)
+            {
+              "data" => {
+                "attributes" => payload
+              }
+            }
+          end
+
+          def skippable_tests_response(payload)
+            skippable_tests = payload.fetch("skippableTests", {}) || {}
+
+            data = skippable_tests.each_value.flat_map do |tests_hash|
+              tests_hash.each_value.flat_map do |test_configs|
+                test_configs.map do |test_config|
+                  {
+                    "type" => Ext::Test::ITR_TEST_SKIPPING_MODE,
+                    "attributes" => {
+                      "suite" => test_config["suite"],
+                      "name" => test_config["name"],
+                      "parameters" => test_config["parameters"]
+                    }
+                  }
+                end
+              end
+            end
+
+            {
+              "meta" => {
+                "correlation_id" => payload["correlationId"]
+              },
+              "data" => data
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimization_cache/readers/missing.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      module Readers
+        class Missing < Base
+          def available?
+            false
+          end
+
+          def load_settings
+            nil
+          end
+
+          def load_known_tests
+            nil
+          end
+
+          def load_test_management
+            nil
+          end
+
+          def load_skippable_tests
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_optimization_cache/readers/v1.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/test_optimization_cache"
+require_relative "base"
+
+module Datadog
+  module CI
+    module TestOptimizationCache
+      module Readers
+        class V1 < Base
+          def initialize(test_optimization_path)
+            @test_optimization_path = test_optimization_path
+          end
+
+          def available?
+            File.exist?(settings_file_path)
+          end
+
+          def load_settings
+            load_http_json(Ext::TestOptimizationCache::SETTINGS_FILE_NAME)
+          end
+
+          def load_known_tests
+            load_http_json(Ext::TestOptimizationCache::KNOWN_TESTS_FILE_NAME)
+          end
+
+          def load_test_management
+            load_http_json(Ext::TestOptimizationCache::TEST_MANAGEMENT_FILE_NAME)
+          end
+
+          def load_skippable_tests
+            load_http_json(Ext::TestOptimizationCache::SKIPPABLE_TESTS_FILE_NAME)
+          end
+
+          private
+
+          def load_http_json(file_name)
+            read_json_file(File.join(http_cache_path, file_name))
+          end
+
+          def settings_file_path
+            File.join(http_cache_path, Ext::TestOptimizationCache::SETTINGS_FILE_NAME)
+          end
+
+          def http_cache_path
+            File.join(
+              @test_optimization_path,
+              Ext::TestOptimizationCache::CACHE_FOLDER_NAME,
+              Ext::TestOptimizationCache::HTTP_CACHE_FOLDER_NAME
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_tracing/component.rb

@@ -225,28 +225,21 @@ module Datadog
         end
 
         def restore_state_from_datadog_test_runner
-          Datadog.logger.debug { "Restoring known tests from DDTest cache" }
+          Datadog.logger.debug { "Restoring known tests from Test Optimization cache" }
 
-          known_tests_data = load_json(Ext::DDTest::KNOWN_TESTS_FILE_NAME)
+          known_tests_data = load_cached_known_tests
           if known_tests_data.nil?
             Datadog.logger.debug { "Restoring known tests failed, will request again" }
             return false
           end
 
-          Datadog.logger.debug { "Restored known tests from DDTest: #{known_tests_data}" }
-
-          # Use the KnownTests class method to parse the JSON data
-          known_tests_data = {
-            "data" => {
-              "attributes" => known_tests_data
-            }
-          }
+          Datadog.logger.debug { "Restored known tests from Test Optimization: #{known_tests_data}" }
 
           @known_tests = KnownTests::Response.from_json(known_tests_data).tests
           @known_tests_enabled = !@known_tests.empty?
 
           unless @known_tests_enabled
-            Datadog.logger.debug("Empty set of known tests from the DDTest cache file")
+            Datadog.logger.debug("Empty set of known tests from the Test Optimization cache file")
           end
 
           Datadog.logger.debug { "Found [#{@known_tests.size}] known tests from context" }

data/lib/datadog/ci/test_tracing/serializers/base.rb

@@ -3,6 +3,7 @@
 require "set"
 
 require_relative "../../ext/test"
+require_relative "meta_truncation"
 
 module Datadog
   module CI
@@ -29,7 +30,9 @@ module Datadog
             @span = span
             @options = options
 
-            @meta = @span.meta.reject { |key, _| Ext::Test::TRANSIENT_TAGS.include?(key) }
+            @meta = MetaTruncation.truncate_string_values(
+              @span.meta.reject { |key, _| Ext::Test::TRANSIENT_TAGS.include?(key) }
+            )
 
             @errors = {}
             @validated = false

data/lib/datadog/ci/test_tracing/serializers/meta_truncation.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module TestTracing
+      module Serializers
+        module MetaTruncation
+          MAX_META_STRING_LENGTH = 5000
+
+          def self.truncate_value(value)
+            return value unless value.is_a?(String) && value.length > MAX_META_STRING_LENGTH
+
+            value[0, MAX_META_STRING_LENGTH]
+          end
+
+          def self.truncate_string_values(tags)
+            tags.transform_values { |value| truncate_value(value) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_tracing/transport.rb

@@ -5,6 +5,7 @@ require "datadog/core/telemetry/logging"
 require "datadog/core/utils/only_once"
 
 require_relative "serializers/factories/test_level"
+require_relative "serializers/meta_truncation"
 
 require_relative "../ext/app_types"
 require_relative "../ext/telemetry"
@@ -105,17 +106,17 @@ module Datadog
 
           if dd_env
             packer.write("env")
-            packer.write(dd_env)
+            packer.write(Serializers::MetaTruncation.truncate_value(dd_env))
           end
 
           packer.write("runtime-id")
-          packer.write(Datadog::Core::Environment::Identity.id)
+          packer.write(Serializers::MetaTruncation.truncate_value(Datadog::Core::Environment::Identity.id))
 
           packer.write("language")
-          packer.write(Datadog::Core::Environment::Identity.lang)
+          packer.write(Serializers::MetaTruncation.truncate_value(Datadog::Core::Environment::Identity.lang))
 
           packer.write("library_version")
-          packer.write(Datadog::CI::VERSION::STRING)
+          packer.write(Serializers::MetaTruncation.truncate_value(Datadog::CI::VERSION::STRING))
 
           library_capabilities_tags = Ext::Test::LibraryCapabilities::CAPABILITY_VERSIONS
 
@@ -124,14 +125,16 @@ module Datadog
             packer.write_map_header(2 + library_capabilities_tags.count)
 
             packer.write(Ext::Test::TAG_TEST_SESSION_NAME)
-            packer.write(test_tracing&.logical_test_session_name)
+            packer.write(Serializers::MetaTruncation.truncate_value(test_tracing&.logical_test_session_name))
 
             packer.write(Ext::Test::TAG_USER_PROVIDED_TEST_SERVICE)
-            packer.write(Utils::Configuration.service_name_provided_by_user?.to_s)
+            packer.write(
+              Serializers::MetaTruncation.truncate_value(Utils::Configuration.service_name_provided_by_user?.to_s)
+            )
 
             library_capabilities_tags.each do |tag, value|
               packer.write(tag)
-              packer.write(value)
+              packer.write(Serializers::MetaTruncation.truncate_value(value))
             end
           end
 

data/lib/datadog/ci/utils/json.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Datadog
+  module CI
+    module Utils
+      module Json
+        def self.read_file(file_path)
+          unless File.exist?(file_path)
+            Datadog.logger.debug { "JSON file not found: #{file_path}" }
+            return nil
+          end
+
+          JSON.parse(File.read(file_path))
+        rescue JSON::ParserError, SystemCallError => e
+          Datadog.logger.debug { "Failed to load JSON file #{file_path}: #{e.message}" }
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/utils/stateful.rb

@@ -1,9 +1,6 @@
 # frozen_string_literal: true
 
-require "json"
 require_relative "file_storage"
-require_relative "test_run"
-require_relative "../ext/dd_test"
 
 module Datadog
   module CI
@@ -22,9 +19,8 @@ module Datadog
 
         # Load component state
         def load_component_state
-          # Check for DDTest cache first
-          if TestRun.test_optimization_data_cached?
-            Datadog.logger.debug { "DDTest cache found" }
+          if test_optimization_cache.cache_available?
+            Datadog.logger.debug { "Test Optimization cache found" }
             return true if restore_state_from_datadog_test_runner
           end
 
@@ -60,22 +56,24 @@ module Datadog
           false
         end
 
-        def load_json(file_name)
-          file_path = File.join(Ext::DDTest::TESTOPTIMIZATION_CACHE_PATH, file_name)
+        def load_cached_settings
+          test_optimization_cache.load_settings
+        end
 
-          unless File.exist?(file_path)
-            Datadog.logger.debug { "JSON file not found: #{file_path}" }
-            return nil
-          end
+        def load_cached_known_tests
+          test_optimization_cache.load_known_tests
+        end
+
+        def load_cached_test_management
+          test_optimization_cache.load_test_management
+        end
+
+        def load_cached_skippable_tests
+          test_optimization_cache.load_skippable_tests
+        end
 
-          content = File.read(file_path)
-          JSON.parse(content)
-        rescue JSON::ParserError => e
-          Datadog.logger.debug { "Failed to parse JSON file #{file_path}: #{e.message}" }
-          nil
-        rescue => e
-          Datadog.logger.debug { "Failed to load JSON file #{file_path}: #{e.message}" }
-          nil
+        def test_optimization_cache
+          Datadog.send(:components).test_optimization_cache
         end
       end
     end

data/lib/datadog/ci/utils/test_run.rb

@@ -2,8 +2,6 @@
 
 require "etc"
 
-require_relative "../ext/dd_test"
-
 module Datadog
   module CI
     module Utils
@@ -44,10 +42,6 @@ module Datadog
 
           @virtual_cpu_count = ::Etc.nprocessors
         end
-
-        def self.test_optimization_data_cached?
-          Dir.exist?(Ext::DDTest::TESTOPTIMIZATION_CACHE_PATH)
-        end
       end
     end
   end

data/lib/datadog/ci/version.rb

@@ -4,7 +4,7 @@ module Datadog
   module CI
     module VERSION
       MAJOR = 1
-      MINOR = 30
+      MINOR = 32
       PATCH = 0
       PRE = nil
       BUILD = nil