datadog-ci 0.8.3 → 1.0.0.beta2

data/lib/datadog/ci/test_visibility/recorder.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "datadog/tracing"
+require "datadog/tracing/contrib/component"
 require "datadog/tracing/trace_digest"
 
 require "rbconfig"
@@ -12,13 +13,14 @@ require_relative "../codeowners/parser"
 require_relative "../ext/app_types"
 require_relative "../ext/test"
 require_relative "../ext/environment"
-require_relative "../utils/git"
+require_relative "../git/local_repository"
 
 require_relative "../span"
 require_relative "../test"
 require_relative "../test_session"
 require_relative "../test_module"
 require_relative "../test_suite"
+require_relative "../worker"
 
 module Datadog
   module CI
@@ -29,8 +31,11 @@ module Datadog
         attr_reader :environment_tags, :test_suite_level_visibility_enabled
 
         def initialize(
-          itr:, remote_settings_api:, test_suite_level_visibility_enabled: false,
-          codeowners: Codeowners::Parser.new(Utils::Git.root).parse
+          itr:,
+          remote_settings_api:,
+          git_tree_upload_worker: DummyWorker.new,
+          test_suite_level_visibility_enabled: false,
+          codeowners: Codeowners::Parser.new(Git::LocalRepository.root).parse
         )
           @test_suite_level_visibility_enabled = test_suite_level_visibility_enabled
 
@@ -42,6 +47,11 @@ module Datadog
 
           @itr = itr
           @remote_settings_api = remote_settings_api
+          @git_tree_upload_worker = git_tree_upload_worker
+        end
+
+        def shutdown!
+          @git_tree_upload_worker.stop
         end
 
         def start_test_session(service: nil, tags: {})
@@ -55,6 +65,7 @@ module Datadog
 
             test_session = build_test_session(tracer_span, tags)
 
+            @git_tree_upload_worker.perform(test_session.git_repository_url)
             configure_library(test_session)
 
             test_session
@@ -116,14 +127,19 @@ module Datadog
               test = build_test(tracer_span, tags)
 
               @local_context.activate_test(test) do
-                block.call(test)
+                on_test_started(test)
+                res = block.call(test)
+                on_test_finished(test)
+                res
               end
             end
           else
             tracer_span = start_datadog_tracer_span(test_name, span_options)
-
             test = build_test(tracer_span, tags)
+
             @local_context.activate_test(test)
+            on_test_started(test)
+
             test
           end
         end
@@ -168,10 +184,16 @@ module Datadog
         end
 
         def deactivate_test
+          test = active_test
+          on_test_finished(test) if test
+
           @local_context.deactivate_test
         end
 
         def deactivate_test_session
+          test_session = active_test_session
+          on_test_session_finished(test_session) if test_session
+
           @global_context.deactivate_test_session!
         end
 
@@ -194,16 +216,34 @@ module Datadog
           return unless itr_enabled?
 
           remote_configuration = @remote_settings_api.fetch_library_settings(test_session)
-          @itr.configure(remote_configuration.payload, test_session)
+          # sometimes we can skip code coverage for default branch if there are no changes in the repository
+          # backend needs git metadata uploaded for this test session to check if we can skip code coverage
+          if remote_configuration.require_git?
+            Datadog.logger.debug { "Library configuration endpoint requires git upload to be finished, waiting..." }
+            @git_tree_upload_worker.wait_until_done
+
+            Datadog.logger.debug { "Requesting library configuration again..." }
+            remote_configuration = @remote_settings_api.fetch_library_settings(test_session)
+
+            if remote_configuration.require_git?
+              Datadog.logger.debug { "git metadata upload did not complete in time when configuring library" }
+            end
+          end
+
+          @itr.configure(
+            remote_configuration.payload,
+            test_session: test_session,
+            git_tree_upload_worker: @git_tree_upload_worker
+          )
         end
 
         def skip_tracing(block = nil)
-          block.call(nil) if block
+          block&.call(nil)
         end
 
         # Sets trace's origin to ciapp-test
         def set_trace_origin(trace)
-          trace.origin = Ext::Test::CONTEXT_ORIGIN if trace
+          trace&.origin = Ext::Test::CONTEXT_ORIGIN
         end
 
         def build_test_session(tracer_span, tags)
@@ -357,6 +397,21 @@ module Datadog
             end
           end
         end
+
+        # TODO: use kind of event system to notify about test finished?
+        def on_test_finished(test)
+          @itr.stop_coverage(test)
+          @itr.count_skipped_test(test)
+        end
+
+        def on_test_started(test)
+          @itr.mark_if_skippable(test)
+          @itr.start_coverage(test)
+        end
+
+        def on_test_session_finished(test_session)
+          @itr.write_test_session_tags(test_session)
+        end
       end
     end
   end

data/lib/datadog/ci/test_visibility/serializers/base.rb

@@ -241,7 +241,7 @@ module Datadog
           end
 
           def to_integer(value)
-            value.to_i if value
+            value&.to_i
           end
         end
       end

data/lib/datadog/ci/test_visibility/serializers/factories/test_level.rb

@@ -8,7 +8,7 @@ module Datadog
     module TestVisibility
       module Serializers
         module Factories
-          # This factory takes care of creating citestcycle serializers when test-level visibility is enabled
+          # This factory takes care of creating msgpack serializers when test-level visibility is enabled
           # NOTE: citestcycle is a protocol Datadog uses to submit test execution tracing information to CI visibility
           # backend
           module TestLevel

data/lib/datadog/ci/test_visibility/serializers/factories/test_suite_level.rb

@@ -11,7 +11,7 @@ module Datadog
     module TestVisibility
       module Serializers
         module Factories
-          # This factory takes care of creating citestcycle serializers when test-suite-level visibility is enabled
+          # This factory takes care of creating msgpack serializers when test-suite-level visibility is enabled
           module TestSuiteLevel
             module_function
 

data/lib/datadog/ci/test_visibility/transport.rb

@@ -1,64 +1,32 @@
 # frozen_string_literal: true
 
-require "msgpack"
-require "uri"
-
-require "datadog/core/encoding"
 require "datadog/core/environment/identity"
-require "datadog/core/chunker"
 
 require_relative "serializers/factories/test_level"
 require_relative "../ext/transport"
+require_relative "../transport/event_platform_transport"
 
 module Datadog
   module CI
     module TestVisibility
-      class Transport
-        # CI test cycle intake's limit is 5.1MB uncompressed
-        # We will use a bit more conservative value 5MB
-        DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
-
-        attr_reader :serializers_factory,
-          :api,
-          :max_payload_size,
-          :dd_env
+      class Transport < Datadog::CI::Transport::EventPlatformTransport
+        attr_reader :serializers_factory, :dd_env
 
         def initialize(
           api:,
-          dd_env: nil,
+          dd_env:,
           serializers_factory: Datadog::CI::TestVisibility::Serializers::Factories::TestLevel,
           max_payload_size: DEFAULT_MAX_PAYLOAD_SIZE
         )
+          super(api: api, max_payload_size: max_payload_size)
+
           @serializers_factory = serializers_factory
-          @max_payload_size = max_payload_size
           @dd_env = dd_env
-          @api = api
         end
 
+        # this method is needed for compatibility with Datadog::Tracing::Writer that uses this Transport
         def send_traces(traces)
-          return [] if traces.nil? || traces.empty?
-
-          Datadog.logger.debug { "Sending #{traces.count} traces..." }
-
-          encoded_events = encode_traces(traces)
-          if encoded_events.empty?
-            Datadog.logger.debug { "Empty encoded events list, skipping send" }
-            return []
-          end
-
-          responses = []
-          Datadog::Core::Chunker.chunk_by_size(encoded_events, max_payload_size).map do |chunk|
-            encoded_payload = pack_events(chunk)
-            Datadog.logger.debug do
-              "Send chunk of #{chunk.count} events; payload size #{encoded_payload.size}"
-            end
-
-            response = send_payload(encoded_payload)
-
-            responses << response
-          end
-
-          responses
+          send_events(traces)
         end
 
         private
@@ -70,15 +38,9 @@ module Datadog
           )
         end
 
-        def encode_traces(traces)
+        def encode_events(traces)
           traces.flat_map do |trace|
-            spans = trace.spans
-            # TODO: remove condition when 1.0 is released
-            if spans.respond_to?(:filter_map)
-              spans.filter_map { |span| encode_span(trace, span) }
-            else
-              spans.map { |span| encode_span(trace, span) }.reject(&:nil?)
-            end
+            trace.spans.filter_map { |span| encode_span(trace, span) }
           end
         end
 
@@ -107,9 +69,7 @@ module Datadog
           Datadog::Core::Encoding::MsgpackEncoder
         end
 
-        def pack_events(encoded_events)
-          packer = MessagePack::Packer.new
-
+        def write_payload_header(packer)
           packer.write_map_header(3) # Set header with how many elements in the map
 
           packer.write("version")
@@ -137,9 +97,6 @@ module Datadog
           packer.write(Datadog::CI::VERSION::STRING)
 
           packer.write("events")
-          packer.write_array_header(encoded_events.size)
-
-          (packer.buffer.to_a + encoded_events).join
         end
       end
     end

data/lib/datadog/ci/transport/api/agentless.rb

@@ -10,10 +10,11 @@ module Datadog
         class Agentless < Base
           attr_reader :api_key
 
-          def initialize(api_key:, citestcycle_url:, api_url:)
+          def initialize(api_key:, citestcycle_url:, api_url:, citestcov_url:)
             @api_key = api_key
             @citestcycle_http = build_http_client(citestcycle_url, compress: true)
             @api_http = build_http_client(api_url, compress: false)
+            @citestcov_http = build_http_client(citestcov_url, compress: true)
           end
 
           def citestcycle_request(path:, payload:, headers: {}, verb: "post")
@@ -28,6 +29,12 @@ module Datadog
             perform_request(@api_http, path: path, payload: payload, headers: headers, verb: verb)
           end
 
+          def citestcov_request(path:, payload:, headers: {}, verb: "post")
+            super(path: path, payload: payload, headers: headers, verb: verb)
+
+            perform_request(@citestcov_http, path: path, payload: @citestcov_payload, headers: headers, verb: verb)
+          end
+
           private
 
           def perform_request(http_client, path:, payload:, headers:, verb:)

data/lib/datadog/ci/transport/api/base.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "securerandom"
+
 require_relative "../../ext/transport"
 
 module Datadog
@@ -15,6 +17,27 @@ module Datadog
             headers[Ext::Transport::HEADER_CONTENT_TYPE] ||= Ext::Transport::CONTENT_TYPE_MESSAGEPACK
           end
 
+          def citestcov_request(path:, payload:, headers: {}, verb: "post")
+            citestcov_request_boundary = ::SecureRandom.uuid
+
+            headers[Ext::Transport::HEADER_CONTENT_TYPE] ||=
+              "#{Ext::Transport::CONTENT_TYPE_MULTIPART_FORM_DATA}; boundary=#{citestcov_request_boundary}"
+
+            @citestcov_payload = [
+              "--#{citestcov_request_boundary}",
+              'Content-Disposition: form-data; name="event"; filename="event.json"',
+              "Content-Type: application/json",
+              "",
+              '{"dummy":true}',
+              "--#{citestcov_request_boundary}",
+              'Content-Disposition: form-data; name="coverage1"; filename="coverage1.msgpack"',
+              "Content-Type: application/msgpack",
+              "",
+              payload,
+              "--#{citestcov_request_boundary}--"
+            ].join("\r\n")
+          end
+
           def headers_with_default(headers)
             request_headers = default_headers
             request_headers.merge!(headers)

data/lib/datadog/ci/transport/api/builder.rb

@@ -24,7 +24,15 @@ module Datadog
             api_url = settings.ci.agentless_url ||
               "https://#{Ext::Transport::DD_API_HOST_PREFIX}.#{dd_site}:443"
 
-            Agentless.new(api_key: settings.api_key, citestcycle_url: citestcycle_url, api_url: api_url)
+            citestcov_url = settings.ci.agentless_url ||
+              "https://#{Ext::Transport::TEST_COVERAGE_INTAKE_HOST_PREFIX}.#{dd_site}:443"
+
+            Agentless.new(
+              api_key: settings.api_key,
+              citestcycle_url: citestcycle_url,
+              api_url: api_url,
+              citestcov_url: citestcov_url
+            )
           end
 
           def self.build_evp_proxy_api(settings)

data/lib/datadog/ci/transport/api/evp_proxy.rb

@@ -38,6 +38,14 @@ module Datadog
             perform_request(@agent_api_http, path: path, payload: payload, headers: headers, verb: verb)
           end
 
+          def citestcov_request(path:, payload:, headers: {}, verb: "post")
+            super(path: path, payload: payload, headers: headers, verb: verb)
+
+            headers[Ext::Transport::HEADER_EVP_SUBDOMAIN] = Ext::Transport::TEST_COVERAGE_INTAKE_HOST_PREFIX
+
+            perform_request(@agent_intake_http, path: path, payload: @citestcov_payload, headers: headers, verb: verb)
+          end
+
           private
 
           def perform_request(http_client, path:, payload:, headers:, verb:)

data/lib/datadog/ci/transport/event_platform_transport.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "msgpack"
+
+require "datadog/core/encoding"
+require "datadog/core/chunker"
+
+module Datadog
+  module CI
+    module Transport
+      class EventPlatformTransport
+        DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
+
+        attr_reader :api,
+          :max_payload_size
+
+        def initialize(api:, max_payload_size: DEFAULT_MAX_PAYLOAD_SIZE)
+          @api = api
+          @max_payload_size = max_payload_size
+        end
+
+        def send_events(events)
+          return [] if events.nil? || events.empty?
+
+          Datadog.logger.debug { "[#{self.class.name}] Sending #{events.count} events..." }
+
+          encoded_events = encode_events(events)
+          if encoded_events.empty?
+            Datadog.logger.debug { "[#{self.class.name}] Empty encoded events list, skipping send" }
+            return []
+          end
+
+          responses = []
+
+          Datadog::Core::Chunker.chunk_by_size(encoded_events, max_payload_size).map do |chunk|
+            encoded_payload = pack_events(chunk)
+            Datadog.logger.debug do
+              "[#{self.class.name}] Send chunk of #{chunk.count} events; payload size #{encoded_payload.size}"
+            end
+
+            response = send_payload(encoded_payload)
+
+            responses << response
+          end
+
+          responses
+        end
+
+        private
+
+        def encoder
+          Datadog::Core::Encoding::MsgpackEncoder
+        end
+
+        def pack_events(encoded_events)
+          packer = MessagePack::Packer.new
+
+          write_payload_header(packer)
+
+          packer.write_array_header(encoded_events.count)
+          (packer.buffer.to_a + encoded_events).join
+        end
+
+        def event_too_large?(event, encoded_event)
+          return false unless encoded_event.size > max_payload_size
+
+          # This single event is too large, we can't flush it
+          Datadog.logger.warn("[#{self.class.name}] Dropping coverage event. Payload too large: '#{event.inspect}'")
+          Datadog.logger.warn(encoded_event)
+
+          true
+        end
+
+        def send_payload(encoded_payload)
+          raise NotImplementedError
+        end
+
+        def encode_events(events)
+          raise NotImplementedError
+        end
+
+        def write_payload_header(packer)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/transport/http.rb

@@ -4,6 +4,7 @@ require "delegate"
 require "datadog/core/transport/http/adapters/net"
 require "datadog/core/transport/http/env"
 require "datadog/core/transport/request"
+require "socket"
 
 require_relative "gzip"
 require_relative "../ext/transport"
@@ -20,6 +21,8 @@ module Datadog
           :compress
 
         DEFAULT_TIMEOUT = 30
+        MAX_RETRIES = 3
+        INITIAL_BACKOFF = 1
 
         def initialize(host:, timeout: DEFAULT_TIMEOUT, port: nil, ssl: true, compress: false)
           @host = host
@@ -29,7 +32,7 @@ module Datadog
           @compress = compress.nil? ? false : compress
         end
 
-        def request(path:, payload:, headers:, verb: "post")
+        def request(path:, payload:, headers:, verb: "post", retries: MAX_RETRIES, backoff: INITIAL_BACKOFF)
           if compress
             headers[Ext::Transport::HEADER_CONTENT_ENCODING] = Ext::Transport::CONTENT_ENCODING_GZIP
             payload = Gzip.compress(payload)
@@ -41,9 +44,7 @@ module Datadog
           end
 
           response = ResponseDecorator.new(
-            adapter.call(
-              build_env(path: path, payload: payload, headers: headers, verb: verb)
-            )
+            perform_http_call(path: path, payload: payload, headers: headers, verb: verb, retries: retries, backoff: backoff)
           )
 
           Datadog.logger.debug do
@@ -55,6 +56,25 @@ module Datadog
 
         private
 
+        def perform_http_call(path:, payload:, headers:, verb:, retries: MAX_RETRIES, backoff: INITIAL_BACKOFF)
+          adapter.call(
+            build_env(path: path, payload: payload, headers: headers, verb: verb)
+          )
+        rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError, SocketError, Net::HTTPBadResponse => e
+          Datadog.logger.debug("Failed to send request with #{e} (#{e.message})")
+
+          if retries.positive?
+            sleep(backoff)
+
+            perform_http_call(
+              path: path, payload: payload, headers: headers, verb: verb, retries: retries - 1, backoff: backoff * 2
+            )
+          else
+            Datadog.logger.error("Failed to send request after #{MAX_RETRIES} retries")
+            raise e
+          end
+        end
+
         def build_env(path:, payload:, headers:, verb:)
           env = Datadog::Core::Transport::HTTP::Env.new(
             Datadog::Core::Transport::Request.new
@@ -67,16 +87,33 @@ module Datadog
         end
 
         def adapter
-          @adapter ||= Datadog::Core::Transport::HTTP::Adapters::Net.new(host, port, timeout: timeout, ssl: ssl)
+          settings = AdapterSettings.new(hostname: host, port: port, ssl: ssl, timeout_seconds: timeout)
+          @adapter ||= Datadog::Core::Transport::HTTP::Adapters::Net.new(settings)
         end
 
         # this is needed because Datadog::Tracing::Writer is not fully compatiple with Datadog::Core::Transport
-        # TODO: remove before 1.0 when CI implements its own worker
+        # TODO: remove when CI implements its own worker
         class ResponseDecorator < ::SimpleDelegator
           def trace_count
             0
           end
         end
+
+        class AdapterSettings
+          attr_reader :hostname, :port, :ssl, :timeout_seconds
+
+          def initialize(hostname:, port: nil, ssl: true, timeout_seconds: nil)
+            @hostname = hostname
+            @port = port
+            @ssl = ssl
+            @timeout_seconds = timeout_seconds
+          end
+
+          def ==(other)
+            hostname == other.hostname && port == other.port && ssl == other.ssl &&
+              timeout_seconds == other.timeout_seconds
+          end
+        end
       end
     end
   end

data/lib/datadog/ci/transport/remote_settings_api.rb

@@ -5,6 +5,7 @@ require "json"
 require "datadog/core/environment/identity"
 
 require_relative "../ext/transport"
+require_relative "../utils/parsing"
 
 module Datadog
   module CI
@@ -28,7 +29,7 @@ module Datadog
             return cached unless cached.nil?
 
             resp = @http_response
-            return @json = default_payload if resp.nil? || !resp.ok?
+            return @json = default_payload if resp.nil? || !ok?
 
             begin
               @json = JSON.parse(resp.payload).dig(*Ext::Transport::DD_API_SETTINGS_RESPONSE_DIG_KEYS) ||
@@ -39,6 +40,10 @@ module Datadog
             end
           end
 
+          def require_git?
+            Utils::Parsing.convert_to_bool(payload[Ext::Transport::DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY])
+          end
+
           private
 
           def default_payload
@@ -46,7 +51,7 @@ module Datadog
           end
         end
 
-        def initialize(api: nil, dd_env: nil)
+        def initialize(dd_env:, api: nil)
           @api = api
           @dd_env = dd_env
         end
@@ -81,10 +86,11 @@ module Datadog
                 "sha" => test_session.git_commit_sha,
                 "test_level" => Ext::Test::ITR_TEST_SKIPPING_MODE,
                 "configurations" => {
-                  "os.platform" => test_session.os_platform,
-                  "os.arch" => test_session.os_architecture,
-                  "runtime.name" => test_session.runtime_name,
-                  "runtime.version" => test_session.runtime_version
+                  Ext::Test::TAG_OS_PLATFORM => test_session.os_platform,
+                  Ext::Test::TAG_OS_ARCHITECTURE => test_session.os_architecture,
+                  Ext::Test::TAG_OS_VERSION => test_session.os_version,
+                  Ext::Test::TAG_RUNTIME_NAME => test_session.runtime_name,
+                  Ext::Test::TAG_RUNTIME_VERSION => test_session.runtime_version
                 }
               }
             }

data/lib/datadog/ci/utils/configuration.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
-require_relative "git"
+require_relative "../git/local_repository"
 
 module Datadog
   module CI
     module Utils
       module Configuration
         def self.fetch_service_name(default)
-          Datadog.configuration.service_without_fallback || Git.repository_name || default
+          Datadog.configuration.service_without_fallback || CI::Git::LocalRepository.repository_name || default
         end
       end
     end