datadog-ci 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14828e8c64b8eeb9c04d4ae2563e568b3b920b587f8e97e628896b143219daf0
-  data.tar.gz: b538d180acd0c18e1d8070adfdcd9f44dad65331048619de40db04cd876dfe01
+  metadata.gz: ae9b964bc83f82487cc72d99641bcd08a575c104ec311327d49715d88852d02d
+  data.tar.gz: 7f787507a51166a49e95c9b55049fc68f3b1d0952fa49d7974f857b37b10e56f
 SHA512:
-  metadata.gz: 60f8fda5f530404027669902b194c7e056af834c882adbc2b7d350eb62f7f75c2d63dae23bf6e98a8bebea0cc8ec34d4f5416fe667d1c30dff1fdb7ebb42854d
-  data.tar.gz: 1b2ad405dee8a1dd2ce24c3d5738ddcbc751f85d5aa5c29a287b614dcfc0a4764230a3b53e59967247fd0f64711fd81fc3863e49895098e7664aa81a3bf1b8e6
+  metadata.gz: 5947c6dd42ab0d629d1e7eecd2f116ea2c28e89b657e47e80eb0b127caf21e25b68e182ba5f3039671491f9335b0dda917091cbd43f18f64c9136e3e38ef1239
+  data.tar.gz: 7790297ed83c4bfedc4cfb9c2b9116a540c8f7c3cdc78ccd9b5ece5ecb506be427765851834f8abca4594ff5908ca99c4e7f4cd8ab601fe5c0228cf2c62f8971

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 ## [Unreleased]
 
+## [0.8.0] - 2024-03-08
+
+### Added
+
+* gzip agent payloads support via evp_proxy/v4 ([#123][])
+
+### Changed
+
+* Add note to README on using VCR ([#122][])
+
+### Fixed
+
+* use framework name as test module name to make test fingerprints stable ([#131][])
+
 ## [0.7.0] - 2024-01-26
 
 ### Added
@@ -150,7 +164,8 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 
 * Ruby versions < 2.7 no longer supported ([#8][])
 
-[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v0.7.0...main
+[Unreleased]: https://github.com/DataDog/datadog-ci-rb/compare/v0.8.0...main
+[0.8.0]: https://github.com/DataDog/datadog-ci-rb/compare/v0.7.0...v0.8.0
 [0.7.0]: https://github.com/DataDog/datadog-ci-rb/compare/v0.6.0...v0.7.0
 [0.6.0]: https://github.com/DataDog/datadog-ci-rb/compare/v0.5.1...v0.6.0
 [0.5.1]: https://github.com/DataDog/datadog-ci-rb/compare/v0.5.0...v0.5.1
@@ -213,3 +228,6 @@ Currently test suite level visibility is not used by our instrumentation: it wil
 [#113]: https://github.com/DataDog/datadog-ci-rb/issues/113
 [#114]: https://github.com/DataDog/datadog-ci-rb/issues/114
 [#115]: https://github.com/DataDog/datadog-ci-rb/issues/115
+[#122]: https://github.com/DataDog/datadog-ci-rb/issues/122
+[#123]: https://github.com/DataDog/datadog-ci-rb/issues/123
+[#131]: https://github.com/DataDog/datadog-ci-rb/issues/131

data/README.md

@@ -175,7 +175,7 @@ Webmock accordingly.
 ```ruby
 # when using agentless mode
 # note to use the correct datadog site (e.g. datadoghq.eu, etc)
-WebMock.disable_net_connect!(:allow => "citestcycle-intake.datadoghq.com")
+WebMock.disable_net_connect!(:allow => /datadoghq.com/)
 
 # when using agent
 WebMock.disable_net_connect!(:allow_localhost => true)
@@ -184,6 +184,25 @@ WebMock.disable_net_connect!(:allow_localhost => true)
 WebMock.disable_net_connect!(:allow => "localhost:8126")
 ```
 
+### VCR
+
+[VCR](https://github.com/vcr/vcr) is another popular testing library for HTTP interactions.
+
+It requires additional configuration to correctly work with datadog-ci:
+
+```ruby
+VCR.configure do |config|
+  # ... your usual configuration here ...
+
+  # when using agent
+  config.ignore_hosts "127.0.0.1", "localhost"
+
+  # when using agentless mode
+  # note to use the correct datadog site (e.g. datadoghq.eu, etc)
+  config.ignore_hosts "citestcycle-intake.datadoghq.com", "api.datadoghq.com"
+end
+```
+
 ### Disabling startup logs
 
 Startup logs produce a report of tracing state when the application is initially configured.

data/lib/datadog/ci/configuration/components.rb

@@ -1,10 +1,7 @@
 # frozen_string_literal: true
 
-require "datadog/core/configuration/agent_settings_resolver"
-require "datadog/core/remote/negotiation"
-
-require_relative "../ext/transport"
 require_relative "../ext/settings"
+require_relative "../itr/runner"
 require_relative "../test_visibility/flush"
 require_relative "../test_visibility/recorder"
 require_relative "../test_visibility/null_recorder"
@@ -12,6 +9,7 @@ require_relative "../test_visibility/serializers/factories/test_level"
 require_relative "../test_visibility/serializers/factories/test_suite_level"
 require_relative "../test_visibility/transport"
 require_relative "../transport/api/builder"
+require_relative "../transport/remote_settings_api"
 
 module Datadog
   module CI
@@ -32,18 +30,7 @@ module Datadog
         end
 
         def activate_ci!(settings)
-          test_visibility_transport = nil
-          agent_settings = Datadog::Core::Configuration::AgentSettingsResolver.call(settings)
-
-          if settings.ci.agentless_mode_enabled
-            check_dd_site(settings)
-            test_visibility_transport = build_agentless_transport(settings)
-          elsif can_use_evp_proxy?(settings, agent_settings)
-            test_visibility_transport = build_evp_proxy_transport(settings, agent_settings)
-          else
-            settings.ci.force_test_level_visibility = true
-          end
-
+          # Configure ddtrace library for CI visibility mode
           # Deactivate telemetry
           settings.telemetry.enabled = false
 
@@ -60,62 +47,77 @@ module Datadog
           # Choose user defined TraceFlush or default to CI TraceFlush
           settings.tracing.test_mode.trace_flush = settings.ci.trace_flush || CI::TestVisibility::Flush::Partial.new
 
+          # transport creation
           writer_options = settings.ci.writer_options
-          if test_visibility_transport
-            writer_options[:transport] = test_visibility_transport
+          test_visibility_api = build_test_visibility_api(settings)
+
+          if test_visibility_api
+            writer_options[:transport] = Datadog::CI::TestVisibility::Transport.new(
+              api: test_visibility_api,
+              serializers_factory: serializers_factory(settings),
+              dd_env: settings.env
+            )
             writer_options[:shutdown_timeout] = 60
             writer_options[:buffer_size] = 10_000
 
             settings.tracing.test_mode.async = true
+          else
+            # only legacy APM protocol is supported, so no test suite level visibility
+            settings.ci.force_test_level_visibility = true
+
+            # ITR is not supported with APM protocol
+            settings.ci.itr_enabled = false
           end
 
           settings.tracing.test_mode.writer_options = writer_options
 
-          @ci_recorder = TestVisibility::Recorder.new(
-            test_suite_level_visibility_enabled: !settings.ci.force_test_level_visibility
+          itr = Datadog::CI::ITR::Runner.new(
+            enabled: settings.ci.enabled && settings.ci.itr_enabled
           )
-        end
 
-        def can_use_evp_proxy?(settings, agent_settings)
-          Datadog::Core::Remote::Negotiation.new(settings, agent_settings).endpoint?(
-            Ext::Transport::EVP_PROXY_PATH_PREFIX
+          remote_settings_api = Transport::RemoteSettingsApi.new(
+            api: test_visibility_api,
+            dd_env: settings.env
+          )
+
+          # CI visibility recorder global instance
+          @ci_recorder = TestVisibility::Recorder.new(
+            test_suite_level_visibility_enabled: !settings.ci.force_test_level_visibility,
+            itr: itr,
+            remote_settings_api: remote_settings_api
           )
         end
 
-        def build_agentless_transport(settings)
-          if settings.api_key.nil?
-            # agentless mode is requested but no API key is provided -
-            # we cannot continue and log an error
-            # Tests are running without CI visibility enabled
+        def build_test_visibility_api(settings)
+          if settings.ci.agentless_mode_enabled
+            check_dd_site(settings)
 
-            Datadog.logger.error(
-              "DATADOG CONFIGURATION - CI VISIBILITY - ATTENTION - " \
-              "Agentless mode was enabled but DD_API_KEY is not set: CI visibility is disabled. " \
-              "Please make sure to set valid api key in DD_API_KEY environment variable"
-            )
+            Datadog.logger.debug("CI visibility configured to use agentless transport")
 
-            settings.ci.enabled = false
+            api = Transport::Api::Builder.build_agentless_api(settings)
+            if api.nil?
+              Datadog.logger.error do
+                "DATADOG CONFIGURATION - CI VISIBILITY - ATTENTION - " \
+                "Agentless mode was enabled but DD_API_KEY is not set: CI visibility is disabled. " \
+                "Please make sure to set valid api key in DD_API_KEY environment variable"
+              end
 
-            nil
-          else
-            Datadog.logger.debug("CI visibility configured to use agentless transport")
+              # Tests are running without CI visibility enabled
+              settings.ci.enabled = false
+            end
 
-            Datadog::CI::TestVisibility::Transport.new(
-              api: Transport::Api::Builder.build_ci_test_cycle_api(settings),
-              serializers_factory: serializers_factory(settings),
-              dd_env: settings.env
-            )
+          else
+            Datadog.logger.debug("CI visibility configured to use agent transport via EVP proxy")
+
+            api = Transport::Api::Builder.build_evp_proxy_api(settings)
+            if api.nil?
+              Datadog.logger.debug(
+                "Old agent version detected, no evp_proxy support. Forcing test level visibility mode"
+              )
+            end
           end
-        end
-
-        def build_evp_proxy_transport(settings, agent_settings)
-          Datadog.logger.debug("CI visibility configured to use agent transport via EVP proxy")
 
-          Datadog::CI::TestVisibility::Transport.new(
-            api: Transport::Api::Builder.build_evp_proxy_api(agent_settings),
-            serializers_factory: serializers_factory(settings),
-            dd_env: settings.env
-          )
+          api
         end
 
         def serializers_factory(settings)
@@ -130,11 +132,11 @@ module Datadog
           return if settings.site.nil?
           return if Ext::Settings::DD_SITE_ALLOWLIST.include?(settings.site)
 
-          Datadog.logger.warn(
+          Datadog.logger.warn do
             "CI VISIBILITY CONFIGURATION " \
             "Agentless mode was enabled but DD_SITE is not set to one of the following: #{Ext::Settings::DD_SITE_ALLOWLIST.join(", ")}. " \
             "Please make sure to set valid site in DD_SITE environment variable"
-          )
+          end
         end
       end
     end

data/lib/datadog/ci/configuration/settings.rb

@@ -55,6 +55,12 @@ module Datadog
                 end
               end
 
+              option :itr_enabled do |o|
+                o.type :bool
+                o.env CI::Ext::Settings::ENV_ITR_ENABLED
+                o.default false
+              end
+
               define_method(:instrument) do |integration_name, options = {}, &block|
                 return unless enabled
 

data/lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -34,14 +34,14 @@ module Datadog
           end
 
           def on_test_run_started(event)
-            test_session = CI.start_test_session(
+            CI.start_test_session(
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                 CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s
               },
               service: configuration[:service_name]
             )
-            CI.start_test_module(test_session.name) if test_session
+            CI.start_test_module(Ext::FRAMEWORK)
           end
 
           def on_test_run_finished(event)

data/lib/datadog/ci/contrib/minitest/runner.rb

@@ -18,14 +18,14 @@ module Datadog
 
               return unless datadog_configuration[:enabled]
 
-              test_session = CI.start_test_session(
+              CI.start_test_session(
                 tags: {
                   CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
                   CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s
                 },
                 service: datadog_configuration[:service_name]
               )
-              CI.start_test_module(test_session.name) if test_session
+              CI.start_test_module(Ext::FRAMEWORK)
             end
 
             private

data/lib/datadog/ci/contrib/rspec/runner.rb

@@ -25,13 +25,12 @@ module Datadog
                 service: datadog_configuration[:service_name]
               )
 
-              test_module = CI.start_test_module(test_session.name) if test_session
+              test_module = CI.start_test_module(Ext::FRAMEWORK)
 
               result = super
               return result unless test_module && test_session
 
               if result != 0
-                # TODO: repeating this twice feels clunky, we need to remove test_module API before GA
                 test_module.failed!
                 test_session.failed!
               else

data/lib/datadog/ci/ext/settings.rb

@@ -10,6 +10,7 @@ module Datadog
         ENV_AGENTLESS_URL = "DD_CIVISIBILITY_AGENTLESS_URL"
         ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED = "DD_CIVISIBILITY_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED"
         ENV_FORCE_TEST_LEVEL_VISIBILITY = "DD_CIVISIBILITY_FORCE_TEST_LEVEL_VISIBILITY"
+        ENV_ITR_ENABLED = "DD_CIVISIBILITY_ITR_ENABLED"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = [

data/lib/datadog/ci/ext/test.rb

@@ -23,6 +23,13 @@ module Datadog
         TAG_CODEOWNERS = "test.codeowners"
         TAG_PARAMETERS = "test.parameters"
 
+        # ITR tags
+        TAG_ITR_TEST_SKIPPING_ENABLED = "test.itr.tests_skipping.enabled"
+        TAG_ITR_TEST_SKIPPING_TYPE = "test.itr.tests_skipping.type"
+
+        # Code coverage tags
+        TAG_CODE_COVERAGE_ENABLED = "test.code_coverage.enabled"
+
         # those tags are special and used to correlate tests with the test sessions, suites, and modules
         # they are transient and not sent to the backend
         TAG_TEST_SESSION_ID = "_test.session_id"
@@ -43,6 +50,10 @@ module Datadog
         TAG_SPAN_KIND = "span.kind"
         SPAN_KIND_TEST = "test"
 
+        # could be either "test" or "suite" depending on whether we skip individual tests or whole suites
+        # we use test skipping for Ruby
+        ITR_TEST_SKIPPING_MODE = "test"
+
         # test status as recognized by Datadog
         module Status
           PASS = "pass"

data/lib/datadog/ci/ext/transport.rb

@@ -12,11 +12,29 @@ module Datadog
         HEADER_EVP_SUBDOMAIN = "X-Datadog-EVP-Subdomain"
         HEADER_CONTAINER_ID = "Datadog-Container-ID"
 
-        EVP_PROXY_PATH_PREFIX = "/evp_proxy/v2/"
+        EVP_PROXY_V2_PATH_PREFIX = "/evp_proxy/v2/"
+        EVP_PROXY_V4_PATH_PREFIX = "/evp_proxy/v4/"
+        EVP_PROXY_PATH_PREFIXES = [EVP_PROXY_V4_PATH_PREFIX, EVP_PROXY_V2_PATH_PREFIX].freeze
+        EVP_PROXY_COMPRESSION_SUPPORTED = {
+          EVP_PROXY_V4_PATH_PREFIX => true,
+          EVP_PROXY_V2_PATH_PREFIX => false
+        }
+
         TEST_VISIBILITY_INTAKE_HOST_PREFIX = "citestcycle-intake"
         TEST_VISIBILITY_INTAKE_PATH = "/api/v2/citestcycle"
 
+        DD_API_HOST_PREFIX = "api"
+        DD_API_SETTINGS_PATH = "/api/v2/libraries/tests/services/setting"
+        DD_API_SETTINGS_TYPE = "ci_app_test_service_libraries_settings"
+        DD_API_SETTINGS_RESPONSE_DIG_KEYS = %w[data attributes].freeze
+        DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY = "itr_enabled"
+        DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY = "code_coverage"
+        DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY = "tests_skipping"
+        DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY = "require_git"
+        DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze
+
         CONTENT_TYPE_MESSAGEPACK = "application/msgpack"
+        CONTENT_TYPE_JSON = "application/json"
         CONTENT_ENCODING_GZIP = "gzip"
       end
     end

data/lib/datadog/ci/itr/runner.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative "../ext/test"
+require_relative "../ext/transport"
+
+module Datadog
+  module CI
+    module ITR
+      # Intelligent test runner implementation
+      # Integrates with backend to provide test impact analysis data and
+      # skip tests that are not impacted by the changes
+      class Runner
+        def initialize(
+          enabled: false
+        )
+          @enabled = enabled
+          @test_skipping_enabled = false
+          @code_coverage_enabled = false
+
+          Datadog.logger.debug("ITR Runner initialized with enabled: #{@enabled}")
+        end
+
+        def configure(remote_configuration, test_session)
+          Datadog.logger.debug("Configuring ITR Runner with remote configuration: #{remote_configuration}")
+
+          @enabled = convert_to_bool(
+            remote_configuration.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY, false)
+          )
+          @test_skipping_enabled = @enabled && convert_to_bool(
+            remote_configuration.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY, false)
+          )
+          @code_coverage_enabled = @enabled && convert_to_bool(
+            remote_configuration.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY, false)
+          )
+
+          test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_ENABLED, @test_skipping_enabled)
+          # currently we set this tag when ITR requires collecting code coverage
+          # this will change as soon as we implement total code coverage support in this library
+          test_session.set_tag(Ext::Test::TAG_CODE_COVERAGE_ENABLED, @code_coverage_enabled)
+
+          # we skip tests, not suites
+          test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_TYPE, Ext::Test::ITR_TEST_SKIPPING_MODE)
+
+          Datadog.logger.debug("Configured ITR Runner with enabled: #{@enabled}, skipping_tests: #{@test_skipping_enabled}, code_coverage: #{@code_coverage_enabled}")
+        end
+
+        def enabled?
+          @enabled
+        end
+
+        def skipping_tests?
+          @test_skipping_enabled
+        end
+
+        def code_coverage?
+          @code_coverage_enabled
+        end
+
+        private
+
+        def convert_to_bool(value)
+          value.to_s == "true"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/span.rb

@@ -121,6 +121,48 @@ module Datadog
         tracer_span.set_tags(tags)
       end
 
+      # Returns the git repository URL extracted from the environment.
+      # @return [String] the repository URL.
+      def git_repository_url
+        tracer_span.get_tag(Ext::Git::TAG_REPOSITORY_URL)
+      end
+
+      # Returns the latest commit SHA extracted from the environment.
+      # @return [String] the commit SHA of the last commit.
+      def git_commit_sha
+        tracer_span.get_tag(Ext::Git::TAG_COMMIT_SHA)
+      end
+
+      # Returns the git branch name extracted from the environment.
+      # @return [String] the branch.
+      def git_branch
+        tracer_span.get_tag(Ext::Git::TAG_BRANCH)
+      end
+
+      # Returns the OS architecture extracted from the environment.
+      # @return [String] OS arch.
+      def os_architecture
+        tracer_span.get_tag(Ext::Test::TAG_OS_ARCHITECTURE)
+      end
+
+      # Returns the OS platform extracted from the environment.
+      # @return [String] OS platform.
+      def os_platform
+        tracer_span.get_tag(Ext::Test::TAG_OS_PLATFORM)
+      end
+
+      # Returns the runtime name extracted from the environment.
+      # @return [String] runtime name.
+      def runtime_name
+        tracer_span.get_tag(Ext::Test::TAG_RUNTIME_NAME)
+      end
+
+      # Returns the runtime version extracted from the environment.
+      # @return [String] runtime version.
+      def runtime_version
+        tracer_span.get_tag(Ext::Test::TAG_RUNTIME_VERSION)
+      end
+
       def set_environment_runtime_tags
         tracer_span.set_tag(Ext::Test::TAG_OS_ARCHITECTURE, ::RbConfig::CONFIG["host_cpu"])
         tracer_span.set_tag(Ext::Test::TAG_OS_PLATFORM, ::RbConfig::CONFIG["host_os"])

data/lib/datadog/ci/test_session.rb

@@ -26,6 +26,14 @@ module Datadog
         get_tag(Ext::Test::TAG_COMMAND)
       end
 
+      def skipping_tests?
+        get_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_ENABLED) == "true"
+      end
+
+      def code_coverage?
+        get_tag(Ext::Test::TAG_CODE_COVERAGE_ENABLED) == "true"
+      end
+
       # Return the test session tags that could be inherited by sub-spans
       # @return [Hash] the tags to be inherited by sub-spans.
       def inheritable_tags

data/lib/datadog/ci/test_visibility/recorder.rb

@@ -29,7 +29,7 @@ module Datadog
         attr_reader :environment_tags, :test_suite_level_visibility_enabled
 
         def initialize(
-          test_suite_level_visibility_enabled: false,
+          itr:, remote_settings_api:, test_suite_level_visibility_enabled: false,
           codeowners: Codeowners::Parser.new(Utils::Git.root).parse
         )
           @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled
@@ -37,7 +37,11 @@ module Datadog
           @environment_tags = Ext::Environment.tags(ENV).freeze
           @local_context = Context::Local.new
           @global_context = Context::Global.new
+
           @codeowners = codeowners
+
+          @itr = itr
+          @remote_settings_api = remote_settings_api
         end
 
         def start_test_session(service: nil, tags: {})
@@ -49,7 +53,11 @@ module Datadog
             )
             set_session_context(tags, tracer_span)
 
-            build_test_session(tracer_span, tags)
+            test_session = build_test_session(tracer_span, tags)
+
+            configure_library(test_session)
+
+            test_session
           end
         end
 
@@ -175,8 +183,20 @@ module Datadog
           @global_context.deactivate_test_suite!(test_suite_name)
         end
 
+        def itr_enabled?
+          @itr.enabled?
+        end
+
         private
 
+        def configure_library(test_session)
+          # this will change when EFD is implemented
+          return unless itr_enabled?
+
+          remote_configuration = @remote_settings_api.fetch_library_settings(test_session)
+          @itr.configure(remote_configuration.payload, test_session)
+        end
+
         def skip_tracing(block = nil)
           block.call(nil) if block
         end

data/lib/datadog/ci/test_visibility/transport.rb

@@ -55,10 +55,6 @@ module Datadog
 
             response = send_payload(encoded_payload)
 
-            Datadog.logger.debug do
-              "Received server response: #{response.inspect}"
-            end
-
             responses << response
           end
 
@@ -68,7 +64,7 @@ module Datadog
         private
 
         def send_payload(encoded_payload)
-          api.request(
+          api.citestcycle_request(
             path: Datadog::CI::Ext::Transport::TEST_VISIBILITY_INTAKE_PATH,
             payload: encoded_payload
           )

data/lib/datadog/ci/transport/api/agentless.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../../ext/transport"
+
+module Datadog
+  module CI
+    module Transport
+      module Api
+        class Agentless < Base
+          attr_reader :api_key
+
+          def initialize(api_key:, citestcycle_url:, api_url:)
+            @api_key = api_key
+            @citestcycle_http = build_http_client(citestcycle_url, compress: true)
+            @api_http = build_http_client(api_url, compress: false)
+          end
+
+          def citestcycle_request(path:, payload:, headers: {}, verb: "post")
+            super
+
+            perform_request(@citestcycle_http, path: path, payload: payload, headers: headers, verb: verb)
+          end
+
+          def api_request(path:, payload:, headers: {}, verb: "post")
+            super
+
+            perform_request(@api_http, path: path, payload: payload, headers: headers, verb: verb)
+          end
+
+          private
+
+          def perform_request(http_client, path:, payload:, headers:, verb:)
+            http_client.request(
+              path: path,
+              payload: payload,
+              headers: headers_with_default(headers),
+              verb: verb
+            )
+          end
+
+          def build_http_client(url, compress:)
+            uri = URI.parse(url)
+            raise "Invalid agentless mode URL: #{url}" if uri.host.nil?
+
+            Datadog::CI::Transport::HTTP.new(
+              host: uri.host,
+              port: uri.port,
+              ssl: uri.scheme == "https" || uri.port == 443,
+              compress: compress
+            )
+          end
+
+          def default_headers
+            headers = super
+            headers[Ext::Transport::HEADER_DD_API_KEY] = api_key
+            headers
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/transport/api/base.rb

@@ -7,27 +7,23 @@ module Datadog
     module Transport
       module Api
         class Base
-          attr_reader :http
+          def api_request(path:, payload:, headers: {}, verb: "post")
+            headers[Ext::Transport::HEADER_CONTENT_TYPE] ||= Ext::Transport::CONTENT_TYPE_JSON
+          end
 
-          def initialize(http:)
-            @http = http
+          def citestcycle_request(path:, payload:, headers: {}, verb: "post")
+            headers[Ext::Transport::HEADER_CONTENT_TYPE] ||= Ext::Transport::CONTENT_TYPE_MESSAGEPACK
           end
 
-          def request(path:, payload:, verb: "post")
-            http.request(
-              path: path,
-              payload: payload,
-              verb: verb,
-              headers: headers
-            )
+          def headers_with_default(headers)
+            request_headers = default_headers
+            request_headers.merge!(headers)
           end
 
           private
 
-          def headers
-            {
-              Ext::Transport::HEADER_CONTENT_TYPE => Ext::Transport::CONTENT_TYPE_MESSAGEPACK
-            }
+          def default_headers
+            {}
           end
         end
       end

data/lib/datadog/ci/transport/api/builder.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
-require_relative "ci_test_cycle"
+require "datadog/core/configuration/agent_settings_resolver"
+require "datadog/core/remote/negotiation"
+
+require_relative "agentless"
 require_relative "evp_proxy"
 require_relative "../http"
 require_relative "../../ext/transport"
@@ -10,34 +13,34 @@ module Datadog
     module Transport
       module Api
         module Builder
-          def self.build_ci_test_cycle_api(settings)
+          def self.build_agentless_api(settings)
+            return nil if settings.api_key.nil?
+
             dd_site = settings.site || Ext::Transport::DEFAULT_DD_SITE
-            url = settings.ci.agentless_url ||
-              "https://#{Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX}.#{dd_site}:443"
 
-            uri = URI.parse(url)
-            raise "Invalid agentless mode URL: #{url}" if uri.host.nil?
+            citestcycle_url = settings.ci.agentless_url ||
+              "https://#{Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX}.#{dd_site}:443"
 
-            http = Datadog::CI::Transport::HTTP.new(
-              host: uri.host,
-              port: uri.port,
-              ssl: uri.scheme == "https" || uri.port == 443,
-              compress: true
-            )
+            api_url = settings.ci.agentless_url ||
+              "https://#{Ext::Transport::DD_API_HOST_PREFIX}.#{dd_site}:443"
 
-            CiTestCycle.new(api_key: settings.api_key, http: http)
+            Agentless.new(api_key: settings.api_key, citestcycle_url: citestcycle_url, api_url: api_url)
           end
 
-          def self.build_evp_proxy_api(agent_settings)
-            http = Datadog::CI::Transport::HTTP.new(
-              host: agent_settings.hostname,
-              port: agent_settings.port,
-              ssl: agent_settings.ssl,
-              timeout: agent_settings.timeout_seconds,
-              compress: false
-            )
+          def self.build_evp_proxy_api(settings)
+            agent_settings = Datadog::Core::Configuration::AgentSettingsResolver.call(settings)
+            negotiation = Datadog::Core::Remote::Negotiation.new(settings, agent_settings)
+
+            # temporary, remove this when patch will be accepted in Core to make logging configurable
+            negotiation.instance_variable_set(:@logged, {no_config_endpoint: true})
+
+            evp_proxy_path_prefix = Ext::Transport::EVP_PROXY_PATH_PREFIXES.find do |path_prefix|
+              negotiation.endpoint?(path_prefix)
+            end
+
+            return nil if evp_proxy_path_prefix.nil?
 
-            EvpProxy.new(http: http)
+            EvpProxy.new(agent_settings: agent_settings, path_prefix: evp_proxy_path_prefix)
           end
         end
       end

data/lib/datadog/ci/transport/api/evp_proxy.rb

@@ -10,17 +10,48 @@ module Datadog
     module Transport
       module Api
         class EvpProxy < Base
-          def request(path:, payload:, verb: "post")
-            path = "#{Ext::Transport::EVP_PROXY_PATH_PREFIX}#{path.sub(/^\//, "")}"
+          def initialize(agent_settings:, path_prefix: Ext::Transport::EVP_PROXY_V2_PATH_PREFIX)
+            @agent_intake_http = build_http_client(
+              agent_settings,
+              compress: Ext::Transport::EVP_PROXY_COMPRESSION_SUPPORTED[path_prefix]
+            )
+
+            @agent_api_http = build_http_client(agent_settings, compress: false)
+
+            path_prefix = "#{path_prefix}/" unless path_prefix.end_with?("/")
+            @path_prefix = path_prefix
+          end
+
+          def citestcycle_request(path:, payload:, headers: {}, verb: "post")
+            super
+
+            headers[Ext::Transport::HEADER_EVP_SUBDOMAIN] = Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX
+
+            perform_request(@agent_intake_http, path: path, payload: payload, headers: headers, verb: verb)
+          end
+
+          def api_request(path:, payload:, headers: {}, verb: "post")
+            super
+
+            headers[Ext::Transport::HEADER_EVP_SUBDOMAIN] = Ext::Transport::DD_API_HOST_PREFIX
 
-            super(
-              path: path,
+            perform_request(@agent_api_http, path: path, payload: payload, headers: headers, verb: verb)
+          end
+
+          private
+
+          def perform_request(http_client, path:, payload:, headers:, verb:)
+            http_client.request(
+              path: path_with_prefix(path),
               payload: payload,
+              headers: headers_with_default(headers),
               verb: verb
             )
           end
 
-          private
+          def path_with_prefix(path)
+            "#{@path_prefix}#{path.sub(/^\//, "")}"
+          end
 
           def container_id
             return @container_id if defined?(@container_id)
@@ -28,15 +59,24 @@ module Datadog
             @container_id = Datadog::Core::Environment::Container.container_id
           end
 
-          def headers
+          def default_headers
             headers = super
-            headers[Ext::Transport::HEADER_EVP_SUBDOMAIN] = Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX
 
             c_id = container_id
             headers[Ext::Transport::HEADER_CONTAINER_ID] = c_id unless c_id.nil?
 
             headers
           end
+
+          def build_http_client(agent_settings, compress:)
+            Datadog::CI::Transport::HTTP.new(
+              host: agent_settings.hostname,
+              port: agent_settings.port,
+              ssl: agent_settings.ssl,
+              timeout: agent_settings.timeout_seconds,
+              compress: compress
+            )
+          end
         end
       end
     end

data/lib/datadog/ci/transport/http.rb

@@ -40,11 +40,17 @@ module Datadog
               "compression_enabled=#{compress}; path=#{path}; payload_size=#{payload.size}"
           end
 
-          ResponseDecorator.new(
+          response = ResponseDecorator.new(
             adapter.call(
               build_env(path: path, payload: payload, headers: headers, verb: verb)
             )
           )
+
+          Datadog.logger.debug do
+            "Received server response: #{response.inspect}"
+          end
+
+          response
         end
 
         private

data/lib/datadog/ci/transport/remote_settings_api.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "json"
+
+require "datadog/core/environment/identity"
+
+require_relative "../ext/transport"
+
+module Datadog
+  module CI
+    module Transport
+      # Datadog API client
+      # Calls settings endpoint to fetch library settings for given service and env
+      class RemoteSettingsApi
+        class Response
+          def initialize(http_response)
+            @http_response = http_response
+            @json = nil
+          end
+
+          def ok?
+            resp = @http_response
+            !resp.nil? && resp.ok?
+          end
+
+          def payload
+            cached = @json
+            return cached unless cached.nil?
+
+            resp = @http_response
+            return @json = default_payload if resp.nil? || !resp.ok?
+
+            begin
+              @json = JSON.parse(resp.payload).dig(*Ext::Transport::DD_API_SETTINGS_RESPONSE_DIG_KEYS) ||
+                default_payload
+            rescue JSON::ParserError => e
+              Datadog.logger.error("Failed to parse settings response payload: #{e}. Payload was: #{resp.payload}")
+              @json = default_payload
+            end
+          end
+
+          private
+
+          def default_payload
+            Ext::Transport::DD_API_SETTINGS_RESPONSE_DEFAULT
+          end
+        end
+
+        def initialize(api: nil, dd_env: nil)
+          @api = api
+          @dd_env = dd_env
+        end
+
+        def fetch_library_settings(test_session)
+          api = @api
+          return Response.new(nil) unless api
+
+          request_payload = payload(test_session)
+          Datadog.logger.debug("Fetching library settings with request: #{request_payload}")
+
+          http_response = api.api_request(
+            path: Ext::Transport::DD_API_SETTINGS_PATH,
+            payload: request_payload
+          )
+
+          Response.new(http_response)
+        end
+
+        private
+
+        def payload(test_session)
+          {
+            "data" => {
+              "id" => Datadog::Core::Environment::Identity.id,
+              "type" => Ext::Transport::DD_API_SETTINGS_TYPE,
+              "attributes" => {
+                "service" => test_session.service,
+                "env" => @dd_env,
+                "repository_url" => test_session.git_repository_url,
+                "branch" => test_session.git_branch,
+                "sha" => test_session.git_commit_sha,
+                "test_level" => Ext::Test::ITR_TEST_SKIPPING_MODE,
+                "configurations" => {
+                  "os.platform" => test_session.os_platform,
+                  "os.arch" => test_session.os_architecture,
+                  "runtime.name" => test_session.runtime_name,
+                  "runtime.version" => test_session.runtime_version
+                }
+              }
+            }
+          }.to_json
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/version.rb

@@ -4,7 +4,7 @@ module Datadog
   module CI
     module VERSION
       MAJOR = "0"
-      MINOR = "7"
+      MINOR = "8"
       PATCH = "0"
       PRE = nil
       BUILD = nil

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datadog-ci
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Datadog, Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-26 00:00:00.000000000 Z
+date: 2024-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -98,6 +98,7 @@ files:
 - lib/datadog/ci/ext/settings.rb
 - lib/datadog/ci/ext/test.rb
 - lib/datadog/ci/ext/transport.rb
+- lib/datadog/ci/itr/runner.rb
 - lib/datadog/ci/span.rb
 - lib/datadog/ci/test.rb
 - lib/datadog/ci/test_module.rb
@@ -118,12 +119,13 @@ files:
 - lib/datadog/ci/test_visibility/serializers/test_v1.rb
 - lib/datadog/ci/test_visibility/serializers/test_v2.rb
 - lib/datadog/ci/test_visibility/transport.rb
+- lib/datadog/ci/transport/api/agentless.rb
 - lib/datadog/ci/transport/api/base.rb
 - lib/datadog/ci/transport/api/builder.rb
-- lib/datadog/ci/transport/api/ci_test_cycle.rb
 - lib/datadog/ci/transport/api/evp_proxy.rb
 - lib/datadog/ci/transport/gzip.rb
 - lib/datadog/ci/transport/http.rb
+- lib/datadog/ci/transport/remote_settings_api.rb
 - lib/datadog/ci/utils/configuration.rb
 - lib/datadog/ci/utils/git.rb
 - lib/datadog/ci/utils/test_run.rb

data/lib/datadog/ci/transport/api/ci_test_cycle.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "base"
-require_relative "../../ext/transport"
-
-module Datadog
-  module CI
-    module Transport
-      module Api
-        class CiTestCycle < Base
-          attr_reader :api_key
-
-          def initialize(api_key:, http:)
-            @api_key = api_key
-
-            super(http: http)
-          end
-
-          private
-
-          def headers
-            headers = super
-            headers[Ext::Transport::HEADER_DD_API_KEY] = api_key
-            headers
-          end
-        end
-      end
-    end
-  end
-end