datadog-ci 0.6.0 → 0.8.0

data/lib/datadog/ci/test_visibility/recorder.rb

@@ -8,12 +8,13 @@ require "rbconfig"
 require_relative "context/global"
 require_relative "context/local"
 
+require_relative "../codeowners/parser"
 require_relative "../ext/app_types"
 require_relative "../ext/test"
 require_relative "../ext/environment"
+require_relative "../utils/git"
 
 require_relative "../span"
-require_relative "../null_span"
 require_relative "../test"
 require_relative "../test_session"
 require_relative "../test_module"
@@ -27,12 +28,20 @@ module Datadog
       class Recorder
         attr_reader :environment_tags, :test_suite_level_visibility_enabled
 
-        def initialize(test_suite_level_visibility_enabled: false)
+        def initialize(
+          itr:, remote_settings_api:, test_suite_level_visibility_enabled: false,
+          codeowners: Codeowners::Parser.new(Utils::Git.root).parse
+        )
           @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled
 
           @environment_tags = Ext::Environment.tags(ENV).freeze
           @local_context = Context::Local.new
           @global_context = Context::Global.new
+
+          @codeowners = codeowners
+
+          @itr = itr
+          @remote_settings_api = remote_settings_api
         end
 
         def start_test_session(service: nil, tags: {})
@@ -44,7 +53,11 @@ module Datadog
             )
             set_session_context(tags, tracer_span)
 
-            build_test_session(tracer_span, tags)
+            test_session = build_test_session(tracer_span, tags)
+
+            configure_library(test_session)
+
+            test_session
           end
         end
 
@@ -88,6 +101,7 @@ module Datadog
           set_suite_context(tags, name: test_suite_name)
 
           tags[Ext::Test::TAG_NAME] = test_name
+          tags[Ext::Test::TAG_TYPE] ||= Ext::Test::Type::TEST
 
           span_options = build_span_options(
             service,
@@ -101,7 +115,7 @@ module Datadog
             start_datadog_tracer_span(test_name, span_options) do |tracer_span|
               test = build_test(tracer_span, tags)
 
-              @local_context.activate_test!(test) do
+              @local_context.activate_test(test) do
                 block.call(test)
               end
             end
@@ -109,15 +123,15 @@ module Datadog
             tracer_span = start_datadog_tracer_span(test_name, span_options)
 
             test = build_test(tracer_span, tags)
-            @local_context.activate_test!(test)
+            @local_context.activate_test(test)
             test
           end
         end
 
-        def trace(span_type, span_name, tags: {}, &block)
+        def trace(span_name, type: "span", tags: {}, &block)
           span_options = build_span_options(
             nil, # service name is completely optional for custom spans
-            span_type,
+            type,
             {resource: span_name}
           )
 
@@ -153,8 +167,8 @@ module Datadog
           @global_context.active_test_suite(test_suite_name)
         end
 
-        def deactivate_test(test)
-          @local_context.deactivate_test!(test)
+        def deactivate_test
+          @local_context.deactivate_test
         end
 
         def deactivate_test_session
@@ -169,14 +183,22 @@ module Datadog
           @global_context.deactivate_test_suite!(test_suite_name)
         end
 
+        def itr_enabled?
+          @itr.enabled?
+        end
+
         private
 
+        def configure_library(test_session)
+          # this will change when EFD is implemented
+          return unless itr_enabled?
+
+          remote_configuration = @remote_settings_api.fetch_library_settings(test_session)
+          @itr.configure(remote_configuration.payload, test_session)
+        end
+
         def skip_tracing(block = nil)
-          if block
-            block.call(null_span)
-          else
-            null_span
-          end
+          block.call(nil) if block
         end
 
         # Sets trace's origin to ciapp-test
@@ -206,6 +228,8 @@ module Datadog
           test = Test.new(tracer_span)
           set_initial_tags(test, tags)
           validate_test_suite_level_visibility_correctness(test)
+          set_codeowners(test)
+
           test
         end
 
@@ -215,9 +239,9 @@ module Datadog
           span
         end
 
-        def build_span_options(service, span_type, other_options = {})
+        def build_span_options(service, type, other_options = {})
           other_options[:service] = service || @global_context.service
-          other_options[:span_type] = span_type
+          other_options[:type] = type
 
           other_options
         end
@@ -251,6 +275,12 @@ module Datadog
           end
         end
 
+        def set_codeowners(test)
+          source = test.source_file
+          owners = @codeowners.list_owners(source) if source
+          test.set_tag(Ext::Test::TAG_CODEOWNERS, owners) unless owners.nil?
+        end
+
         def set_suite_context(tags, span: nil, name: nil)
           return if span.nil? && name.nil?
 
@@ -280,10 +310,6 @@ module Datadog
           end
         end
 
-        def null_span
-          @null_span ||= NullSpan.new
-        end
-
         def validate_test_suite_level_visibility_correctness(test)
           return unless test_suite_level_visibility_enabled
 

data/lib/datadog/ci/test_visibility/serializers/base.rb

@@ -34,7 +34,7 @@ module Datadog
             @trace = trace
             @span = span
 
-            @meta = @span.meta.reject { |key, _| Ext::Test::SPECIAL_TAGS.include?(key) }
+            @meta = @span.meta.reject { |key, _| Ext::Test::TRANSIENT_TAGS.include?(key) }
 
             @errors = {}
             @validated = false
@@ -45,7 +45,7 @@ module Datadog
 
             packer.write_map_header(3)
 
-            write_field(packer, "type")
+            write_field(packer, "type", "event_type")
             write_field(packer, "version")
 
             packer.write("content")
@@ -119,9 +119,6 @@ module Datadog
             to_integer(@span.get_tag(Ext::Test::TAG_TEST_SUITE_ID))
           end
 
-          def type
-          end
-
           def version
             1
           end
@@ -130,6 +127,10 @@ module Datadog
             @span.type
           end
 
+          def event_type
+            "span"
+          end
+
           def name
             @span.name
           end

data/lib/datadog/ci/test_visibility/serializers/span.rb

@@ -21,7 +21,7 @@ module Datadog
             CONTENT_MAP_SIZE
           end
 
-          def type
+          def event_type
             "span"
           end
 

data/lib/datadog/ci/test_visibility/serializers/test_module.rb

@@ -22,7 +22,7 @@ module Datadog
             CONTENT_MAP_SIZE
           end
 
-          def type
+          def event_type
             Ext::AppTypes::TYPE_TEST_MODULE
           end
 

data/lib/datadog/ci/test_visibility/serializers/test_session.rb

@@ -22,7 +22,7 @@ module Datadog
             CONTENT_MAP_SIZE
           end
 
-          def type
+          def event_type
             Ext::AppTypes::TYPE_TEST_SESSION
           end
 

data/lib/datadog/ci/test_visibility/serializers/test_suite.rb

@@ -22,7 +22,7 @@ module Datadog
             CONTENT_MAP_SIZE
           end
 
-          def type
+          def event_type
             Ext::AppTypes::TYPE_TEST_SUITE
           end
 

data/lib/datadog/ci/test_visibility/serializers/test_v1.rb

@@ -22,7 +22,7 @@ module Datadog
             CONTENT_MAP_SIZE
           end
 
-          def type
+          def event_type
             Ext::AppTypes::TYPE_TEST
           end
 

data/lib/datadog/ci/test_visibility/transport.rb

@@ -55,10 +55,6 @@ module Datadog
 
             response = send_payload(encoded_payload)
 
-            Datadog.logger.debug do
-              "Received server response: #{response.inspect}"
-            end
-
             responses << response
           end
 
@@ -68,7 +64,7 @@ module Datadog
         private
 
         def send_payload(encoded_payload)
-          api.request(
+          api.citestcycle_request(
             path: Datadog::CI::Ext::Transport::TEST_VISIBILITY_INTAKE_PATH,
             payload: encoded_payload
           )

data/lib/datadog/ci/transport/api/agentless.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../../ext/transport"
+
+module Datadog
+  module CI
+    module Transport
+      module Api
+        class Agentless < Base
+          attr_reader :api_key
+
+          def initialize(api_key:, citestcycle_url:, api_url:)
+            @api_key = api_key
+            @citestcycle_http = build_http_client(citestcycle_url, compress: true)
+            @api_http = build_http_client(api_url, compress: false)
+          end
+
+          def citestcycle_request(path:, payload:, headers: {}, verb: "post")
+            super
+
+            perform_request(@citestcycle_http, path: path, payload: payload, headers: headers, verb: verb)
+          end
+
+          def api_request(path:, payload:, headers: {}, verb: "post")
+            super
+
+            perform_request(@api_http, path: path, payload: payload, headers: headers, verb: verb)
+          end
+
+          private
+
+          def perform_request(http_client, path:, payload:, headers:, verb:)
+            http_client.request(
+              path: path,
+              payload: payload,
+              headers: headers_with_default(headers),
+              verb: verb
+            )
+          end
+
+          def build_http_client(url, compress:)
+            uri = URI.parse(url)
+            raise "Invalid agentless mode URL: #{url}" if uri.host.nil?
+
+            Datadog::CI::Transport::HTTP.new(
+              host: uri.host,
+              port: uri.port,
+              ssl: uri.scheme == "https" || uri.port == 443,
+              compress: compress
+            )
+          end
+
+          def default_headers
+            headers = super
+            headers[Ext::Transport::HEADER_DD_API_KEY] = api_key
+            headers
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/transport/api/base.rb

@@ -7,27 +7,23 @@ module Datadog
     module Transport
       module Api
         class Base
-          attr_reader :http
+          def api_request(path:, payload:, headers: {}, verb: "post")
+            headers[Ext::Transport::HEADER_CONTENT_TYPE] ||= Ext::Transport::CONTENT_TYPE_JSON
+          end
 
-          def initialize(http:)
-            @http = http
+          def citestcycle_request(path:, payload:, headers: {}, verb: "post")
+            headers[Ext::Transport::HEADER_CONTENT_TYPE] ||= Ext::Transport::CONTENT_TYPE_MESSAGEPACK
           end
 
-          def request(path:, payload:, verb: "post")
-            http.request(
-              path: path,
-              payload: payload,
-              verb: verb,
-              headers: headers
-            )
+          def headers_with_default(headers)
+            request_headers = default_headers
+            request_headers.merge!(headers)
           end
 
           private
 
-          def headers
-            {
-              Ext::Transport::HEADER_CONTENT_TYPE => Ext::Transport::CONTENT_TYPE_MESSAGEPACK
-            }
+          def default_headers
+            {}
           end
         end
       end

data/lib/datadog/ci/transport/api/builder.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
-require_relative "ci_test_cycle"
+require "datadog/core/configuration/agent_settings_resolver"
+require "datadog/core/remote/negotiation"
+
+require_relative "agentless"
 require_relative "evp_proxy"
 require_relative "../http"
 require_relative "../../ext/transport"
@@ -10,34 +13,34 @@ module Datadog
     module Transport
       module Api
         module Builder
-          def self.build_ci_test_cycle_api(settings)
+          def self.build_agentless_api(settings)
+            return nil if settings.api_key.nil?
+
             dd_site = settings.site || Ext::Transport::DEFAULT_DD_SITE
-            url = settings.ci.agentless_url ||
-              "https://#{Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX}.#{dd_site}:443"
 
-            uri = URI.parse(url)
-            raise "Invalid agentless mode URL: #{url}" if uri.host.nil?
+            citestcycle_url = settings.ci.agentless_url ||
+              "https://#{Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX}.#{dd_site}:443"
 
-            http = Datadog::CI::Transport::HTTP.new(
-              host: uri.host,
-              port: uri.port,
-              ssl: uri.scheme == "https" || uri.port == 443,
-              compress: true
-            )
+            api_url = settings.ci.agentless_url ||
+              "https://#{Ext::Transport::DD_API_HOST_PREFIX}.#{dd_site}:443"
 
-            CiTestCycle.new(api_key: settings.api_key, http: http)
+            Agentless.new(api_key: settings.api_key, citestcycle_url: citestcycle_url, api_url: api_url)
           end
 
-          def self.build_evp_proxy_api(agent_settings)
-            http = Datadog::CI::Transport::HTTP.new(
-              host: agent_settings.hostname,
-              port: agent_settings.port,
-              ssl: agent_settings.ssl,
-              timeout: agent_settings.timeout_seconds,
-              compress: false
-            )
+          def self.build_evp_proxy_api(settings)
+            agent_settings = Datadog::Core::Configuration::AgentSettingsResolver.call(settings)
+            negotiation = Datadog::Core::Remote::Negotiation.new(settings, agent_settings)
+
+            # temporary, remove this when patch will be accepted in Core to make logging configurable
+            negotiation.instance_variable_set(:@logged, {no_config_endpoint: true})
+
+            evp_proxy_path_prefix = Ext::Transport::EVP_PROXY_PATH_PREFIXES.find do |path_prefix|
+              negotiation.endpoint?(path_prefix)
+            end
+
+            return nil if evp_proxy_path_prefix.nil?
 
-            EvpProxy.new(http: http)
+            EvpProxy.new(agent_settings: agent_settings, path_prefix: evp_proxy_path_prefix)
           end
         end
       end

data/lib/datadog/ci/transport/api/evp_proxy.rb

@@ -10,17 +10,48 @@ module Datadog
     module Transport
       module Api
         class EvpProxy < Base
-          def request(path:, payload:, verb: "post")
-            path = "#{Ext::Transport::EVP_PROXY_PATH_PREFIX}#{path.sub(/^\//, "")}"
+          def initialize(agent_settings:, path_prefix: Ext::Transport::EVP_PROXY_V2_PATH_PREFIX)
+            @agent_intake_http = build_http_client(
+              agent_settings,
+              compress: Ext::Transport::EVP_PROXY_COMPRESSION_SUPPORTED[path_prefix]
+            )
+
+            @agent_api_http = build_http_client(agent_settings, compress: false)
+
+            path_prefix = "#{path_prefix}/" unless path_prefix.end_with?("/")
+            @path_prefix = path_prefix
+          end
+
+          def citestcycle_request(path:, payload:, headers: {}, verb: "post")
+            super
+
+            headers[Ext::Transport::HEADER_EVP_SUBDOMAIN] = Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX
+
+            perform_request(@agent_intake_http, path: path, payload: payload, headers: headers, verb: verb)
+          end
+
+          def api_request(path:, payload:, headers: {}, verb: "post")
+            super
+
+            headers[Ext::Transport::HEADER_EVP_SUBDOMAIN] = Ext::Transport::DD_API_HOST_PREFIX
 
-            super(
-              path: path,
+            perform_request(@agent_api_http, path: path, payload: payload, headers: headers, verb: verb)
+          end
+
+          private
+
+          def perform_request(http_client, path:, payload:, headers:, verb:)
+            http_client.request(
+              path: path_with_prefix(path),
               payload: payload,
+              headers: headers_with_default(headers),
               verb: verb
             )
           end
 
-          private
+          def path_with_prefix(path)
+            "#{@path_prefix}#{path.sub(/^\//, "")}"
+          end
 
           def container_id
             return @container_id if defined?(@container_id)
@@ -28,15 +59,24 @@ module Datadog
             @container_id = Datadog::Core::Environment::Container.container_id
           end
 
-          def headers
+          def default_headers
             headers = super
-            headers[Ext::Transport::HEADER_EVP_SUBDOMAIN] = Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX
 
             c_id = container_id
             headers[Ext::Transport::HEADER_CONTAINER_ID] = c_id unless c_id.nil?
 
             headers
           end
+
+          def build_http_client(agent_settings, compress:)
+            Datadog::CI::Transport::HTTP.new(
+              host: agent_settings.hostname,
+              port: agent_settings.port,
+              ssl: agent_settings.ssl,
+              timeout: agent_settings.timeout_seconds,
+              compress: compress
+            )
+          end
         end
       end
     end

data/lib/datadog/ci/transport/http.rb

@@ -40,11 +40,17 @@ module Datadog
               "compression_enabled=#{compress}; path=#{path}; payload_size=#{payload.size}"
           end
 
-          ResponseDecorator.new(
+          response = ResponseDecorator.new(
             adapter.call(
               build_env(path: path, payload: payload, headers: headers, verb: verb)
             )
           )
+
+          Datadog.logger.debug do
+            "Received server response: #{response.inspect}"
+          end
+
+          response
         end
 
         private

data/lib/datadog/ci/transport/remote_settings_api.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "json"
+
+require "datadog/core/environment/identity"
+
+require_relative "../ext/transport"
+
+module Datadog
+  module CI
+    module Transport
+      # Datadog API client
+      # Calls settings endpoint to fetch library settings for given service and env
+      class RemoteSettingsApi
+        class Response
+          def initialize(http_response)
+            @http_response = http_response
+            @json = nil
+          end
+
+          def ok?
+            resp = @http_response
+            !resp.nil? && resp.ok?
+          end
+
+          def payload
+            cached = @json
+            return cached unless cached.nil?
+
+            resp = @http_response
+            return @json = default_payload if resp.nil? || !resp.ok?
+
+            begin
+              @json = JSON.parse(resp.payload).dig(*Ext::Transport::DD_API_SETTINGS_RESPONSE_DIG_KEYS) ||
+                default_payload
+            rescue JSON::ParserError => e
+              Datadog.logger.error("Failed to parse settings response payload: #{e}. Payload was: #{resp.payload}")
+              @json = default_payload
+            end
+          end
+
+          private
+
+          def default_payload
+            Ext::Transport::DD_API_SETTINGS_RESPONSE_DEFAULT
+          end
+        end
+
+        def initialize(api: nil, dd_env: nil)
+          @api = api
+          @dd_env = dd_env
+        end
+
+        def fetch_library_settings(test_session)
+          api = @api
+          return Response.new(nil) unless api
+
+          request_payload = payload(test_session)
+          Datadog.logger.debug("Fetching library settings with request: #{request_payload}")
+
+          http_response = api.api_request(
+            path: Ext::Transport::DD_API_SETTINGS_PATH,
+            payload: request_payload
+          )
+
+          Response.new(http_response)
+        end
+
+        private
+
+        def payload(test_session)
+          {
+            "data" => {
+              "id" => Datadog::Core::Environment::Identity.id,
+              "type" => Ext::Transport::DD_API_SETTINGS_TYPE,
+              "attributes" => {
+                "service" => test_session.service,
+                "env" => @dd_env,
+                "repository_url" => test_session.git_repository_url,
+                "branch" => test_session.git_branch,
+                "sha" => test_session.git_commit_sha,
+                "test_level" => Ext::Test::ITR_TEST_SKIPPING_MODE,
+                "configurations" => {
+                  "os.platform" => test_session.os_platform,
+                  "os.arch" => test_session.os_architecture,
+                  "runtime.name" => test_session.runtime_name,
+                  "runtime.version" => test_session.runtime_version
+                }
+              }
+            }
+          }.to_json
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/utils/configuration.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require_relative "git"
+
+module Datadog
+  module CI
+    module Utils
+      module Configuration
+        def self.fetch_service_name(default)
+          Datadog.configuration.service_without_fallback || Git.repository_name || default
+        end
+      end
+    end
+  end
+end