datadog-ci 0.8.2 → 1.0.0.beta1

data/lib/datadog/ci/ext/environment/providers/base.rb

@@ -103,7 +103,7 @@ module Datadog
               @branch = @tag = nil
 
               # @type var branch_or_tag_string: untyped
-              if branch_or_tag_string && branch_or_tag_string.include?("tags/")
+              if branch_or_tag_string&.include?("tags/")
                 @tag = branch_or_tag_string
               else
                 @branch = branch_or_tag_string

data/lib/datadog/ci/ext/environment/providers/bitbucket.rb

@@ -20,7 +20,7 @@ module Datadog
             end
 
             def pipeline_id
-              env["BITBUCKET_PIPELINE_UUID"] ? env["BITBUCKET_PIPELINE_UUID"].tr("{}", "") : nil
+              env["BITBUCKET_PIPELINE_UUID"]&.tr("{}", "")
             end
 
             def pipeline_name

data/lib/datadog/ci/ext/environment/providers/github_actions.rb

@@ -2,8 +2,9 @@
 
 require "json"
 
+require "datadog/core/utils/url"
+
 require_relative "base"
-require_relative "../../../utils/url"
 
 module Datadog
   module CI
@@ -79,7 +80,7 @@ module Datadog
             def github_server_url
               return @github_server_url if defined?(@github_server_url)
 
-              @github_server_url ||= Utils::Url.filter_sensitive_info(env["GITHUB_SERVER_URL"])
+              @github_server_url ||= Datadog::Core::Utils::Url.filter_basic_auth(env["GITHUB_SERVER_URL"])
             end
           end
         end

data/lib/datadog/ci/ext/environment.rb

@@ -71,7 +71,7 @@ module Datadog
             return
           end
 
-          unless HEX_NUMBER_REGEXP =~ git_sha
+          unless HEX_NUMBER_REGEXP.match?(git_sha)
             message += " Expected SHA to be a valid HEX number, got #{git_sha}."
             Datadog.logger.error(message)
           end

data/lib/datadog/ci/ext/transport.rb

@@ -23,6 +23,9 @@ module Datadog
         TEST_VISIBILITY_INTAKE_HOST_PREFIX = "citestcycle-intake"
         TEST_VISIBILITY_INTAKE_PATH = "/api/v2/citestcycle"
 
+        TEST_COVERAGE_INTAKE_HOST_PREFIX = "citestcov-intake"
+        TEST_COVERAGE_INTAKE_PATH = "/api/v2/citestcov"
+
         DD_API_HOST_PREFIX = "api"
         DD_API_SETTINGS_PATH = "/api/v2/libraries/tests/services/setting"
         DD_API_SETTINGS_TYPE = "ci_app_test_service_libraries_settings"
@@ -35,6 +38,7 @@ module Datadog
 
         CONTENT_TYPE_MESSAGEPACK = "application/msgpack"
         CONTENT_TYPE_JSON = "application/json"
+        CONTENT_TYPE_MULTIPART_FORM_DATA = "multipart/form-data"
         CONTENT_ENCODING_GZIP = "gzip"
       end
     end

data/lib/datadog/ci/itr/coverage/ddcov.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module ITR
+      module Coverage
+        # Placeholder for code coverage collection
+        # Implementation in ext/datadog_cov
+        class DDCov
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/itr/coverage/event.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "msgpack"
+
+module Datadog
+  module CI
+    module ITR
+      module Coverage
+        class Event
+          attr_reader :test_id, :test_suite_id, :test_session_id, :coverage
+
+          def initialize(test_id:, test_suite_id:, test_session_id:, coverage:)
+            @test_id = test_id
+            @test_suite_id = test_suite_id
+            @test_session_id = test_session_id
+            @coverage = coverage
+          end
+
+          def valid?
+            valid = true
+
+            [:test_id, :test_suite_id, :test_session_id, :coverage].each do |key|
+              next unless send(key).nil?
+
+              Datadog.logger.warn("citestcov event is invalid: [#{key}] is nil. Event: #{self}")
+              valid = false
+            end
+
+            valid
+          end
+
+          def to_msgpack(packer = nil)
+            packer ||= MessagePack::Packer.new
+
+            packer.write_map_header(4)
+
+            packer.write("test_session_id")
+            packer.write(test_session_id.to_i)
+
+            packer.write("test_suite_id")
+            packer.write(test_suite_id.to_i)
+
+            packer.write("span_id")
+            packer.write(test_id.to_i)
+
+            files = coverage.keys
+            packer.write("files")
+            packer.write_array_header(files.size)
+
+            files.each do |filename|
+              packer.write_map_header(1)
+              packer.write("filename")
+              packer.write(filename)
+            end
+          end
+
+          def to_s
+            "Coverage::Event[test_id=#{test_id}, test_suite_id=#{test_suite_id}, test_session_id=#{test_session_id}, coverage=#{coverage}]"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/itr/coverage/transport.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "event"
+require_relative "../../transport/event_platform_transport"
+
+module Datadog
+  module CI
+    module ITR
+      module Coverage
+        class Transport < Datadog::CI::Transport::EventPlatformTransport
+          private
+
+          def send_payload(encoded_payload)
+            api.citestcov_request(
+              path: Ext::Transport::TEST_COVERAGE_INTAKE_PATH,
+              payload: encoded_payload
+            )
+          end
+
+          def encode_events(events)
+            events.filter_map do |event|
+              next unless event.valid?
+
+              encoded = encoder.encode(event)
+              next if event_too_large?(event, encoded)
+
+              encoded
+            end
+          end
+
+          def write_payload_header(packer)
+            packer.write_map_header(2)
+            packer.write("version")
+            packer.write(2)
+
+            packer.write("coverages")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/itr/runner.rb

@@ -3,6 +3,8 @@
 require_relative "../ext/test"
 require_relative "../ext/transport"
 
+require_relative "../utils/git"
+
 module Datadog
   module CI
     module ITR
@@ -41,6 +43,8 @@ module Datadog
           # we skip tests, not suites
           test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_TYPE, Ext::Test::ITR_TEST_SKIPPING_MODE)
 
+          load_datadog_cov! if @code_coverage_enabled
+
           Datadog.logger.debug("Configured ITR Runner with enabled: #{@enabled}, skipping_tests: #{@test_skipping_enabled}, code_coverage: #{@code_coverage_enabled}")
         end
 
@@ -56,8 +60,32 @@ module Datadog
           @code_coverage_enabled
         end
 
+        def start_coverage
+          return if !enabled? || !code_coverage?
+
+          coverage_collector&.start
+        end
+
+        def stop_coverage(_test)
+          return if !enabled? || !code_coverage?
+
+          coverage_collector&.stop
+        end
+
         private
 
+        def coverage_collector
+          Thread.current[:dd_coverage_collector] ||= Coverage::DDCov.new(root: Utils::Git.root)
+        end
+
+        def load_datadog_cov!
+          require "datadog_cov.#{RUBY_VERSION}_#{RUBY_PLATFORM}"
+        rescue LoadError => e
+          Datadog.logger.error("Failed to load coverage collector: #{e}. Code coverage will not be collected.")
+
+          @code_coverage_enabled = false
+        end
+
         def convert_to_bool(value)
           value.to_s == "true"
         end

data/lib/datadog/ci/test.rb

@@ -112,8 +112,7 @@ module Datadog
       private
 
       def record_test_result(datadog_status)
-        suite = test_suite
-        suite.record_test_result(datadog_status) if suite
+        test_suite&.record_test_result(datadog_status)
       end
     end
   end

data/lib/datadog/ci/test_visibility/context/global.rb

@@ -55,9 +55,7 @@ module Datadog
 
           def service
             @mutex.synchronize do
-              # thank you RBS for this weirdness
-              test_session = @test_session
-              test_session.service if test_session
+              @test_session&.service
             end
           end
 

data/lib/datadog/ci/test_visibility/null_recorder.rb

@@ -45,7 +45,7 @@ module Datadog
         private
 
         def skip_tracing(block = nil)
-          block.call(nil) if block
+          block&.call(nil)
         end
       end
     end

data/lib/datadog/ci/test_visibility/recorder.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "datadog/tracing"
+require "datadog/tracing/contrib/component"
 require "datadog/tracing/trace_digest"
 
 require "rbconfig"
@@ -116,14 +117,22 @@ module Datadog
               test = build_test(tracer_span, tags)
 
               @local_context.activate_test(test) do
-                block.call(test)
+                @itr.start_coverage
+
+                res = block.call(test)
+                on_test_finished(test)
+
+                res
               end
             end
           else
             tracer_span = start_datadog_tracer_span(test_name, span_options)
 
             test = build_test(tracer_span, tags)
+
             @local_context.activate_test(test)
+            @itr.start_coverage
+
             test
           end
         end
@@ -168,6 +177,9 @@ module Datadog
         end
 
         def deactivate_test
+          test = active_test
+          on_test_finished(test) if test
+
           @local_context.deactivate_test
         end
 
@@ -198,12 +210,12 @@ module Datadog
         end
 
         def skip_tracing(block = nil)
-          block.call(nil) if block
+          block&.call(nil)
         end
 
         # Sets trace's origin to ciapp-test
         def set_trace_origin(trace)
-          trace.origin = Ext::Test::CONTEXT_ORIGIN if trace
+          trace&.origin = Ext::Test::CONTEXT_ORIGIN
         end
 
         def build_test_session(tracer_span, tags)
@@ -357,6 +369,11 @@ module Datadog
             end
           end
         end
+
+        # TODO: use kind of event system to notify about test finished?
+        def on_test_finished(test)
+          @itr.stop_coverage(test)
+        end
       end
     end
   end

data/lib/datadog/ci/test_visibility/serializers/base.rb

@@ -241,7 +241,7 @@ module Datadog
           end
 
           def to_integer(value)
-            value.to_i if value
+            value&.to_i
           end
         end
       end

data/lib/datadog/ci/test_visibility/serializers/factories/test_level.rb

@@ -8,7 +8,7 @@ module Datadog
     module TestVisibility
       module Serializers
         module Factories
-          # This factory takes care of creating citestcycle serializers when test-level visibility is enabled
+          # This factory takes care of creating msgpack serializers when test-level visibility is enabled
           # NOTE: citestcycle is a protocol Datadog uses to submit test execution tracing information to CI visibility
           # backend
           module TestLevel

data/lib/datadog/ci/test_visibility/serializers/factories/test_suite_level.rb

@@ -11,7 +11,7 @@ module Datadog
     module TestVisibility
       module Serializers
         module Factories
-          # This factory takes care of creating citestcycle serializers when test-suite-level visibility is enabled
+          # This factory takes care of creating msgpack serializers when test-suite-level visibility is enabled
           module TestSuiteLevel
             module_function
 

data/lib/datadog/ci/test_visibility/transport.rb

@@ -1,27 +1,16 @@
 # frozen_string_literal: true
 
-require "msgpack"
-require "uri"
-
-require "datadog/core/encoding"
 require "datadog/core/environment/identity"
-require "datadog/core/chunker"
 
 require_relative "serializers/factories/test_level"
 require_relative "../ext/transport"
+require_relative "../transport/event_platform_transport"
 
 module Datadog
   module CI
     module TestVisibility
-      class Transport
-        # CI test cycle intake's limit is 5.1MB uncompressed
-        # We will use a bit more conservative value 5MB
-        DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
-
-        attr_reader :serializers_factory,
-          :api,
-          :max_payload_size,
-          :dd_env
+      class Transport < Datadog::CI::Transport::EventPlatformTransport
+        attr_reader :serializers_factory, :dd_env
 
         def initialize(
           api:,
@@ -29,36 +18,15 @@ module Datadog
           serializers_factory: Datadog::CI::TestVisibility::Serializers::Factories::TestLevel,
           max_payload_size: DEFAULT_MAX_PAYLOAD_SIZE
         )
+          super(api: api, max_payload_size: max_payload_size)
+
           @serializers_factory = serializers_factory
-          @max_payload_size = max_payload_size
           @dd_env = dd_env
-          @api = api
         end
 
+        # this method is needed for compatibility with Datadog::Tracing::Writer that uses this Transport
         def send_traces(traces)
-          return [] if traces.nil? || traces.empty?
-
-          Datadog.logger.debug { "Sending #{traces.count} traces..." }
-
-          encoded_events = encode_traces(traces)
-          if encoded_events.empty?
-            Datadog.logger.debug { "Empty encoded events list, skipping send" }
-            return []
-          end
-
-          responses = []
-          Datadog::Core::Chunker.chunk_by_size(encoded_events, max_payload_size).map do |chunk|
-            encoded_payload = pack_events(chunk)
-            Datadog.logger.debug do
-              "Send chunk of #{chunk.count} events; payload size #{encoded_payload.size}"
-            end
-
-            response = send_payload(encoded_payload)
-
-            responses << response
-          end
-
-          responses
+          send_events(traces)
         end
 
         private
@@ -70,15 +38,9 @@ module Datadog
           )
         end
 
-        def encode_traces(traces)
+        def encode_events(traces)
           traces.flat_map do |trace|
-            spans = trace.spans
-            # TODO: remove condition when 1.0 is released
-            if spans.respond_to?(:filter_map)
-              spans.filter_map { |span| encode_span(trace, span) }
-            else
-              spans.map { |span| encode_span(trace, span) }.reject(&:nil?)
-            end
+            trace.spans.filter_map { |span| encode_span(trace, span) }
           end
         end
 
@@ -107,9 +69,7 @@ module Datadog
           Datadog::Core::Encoding::MsgpackEncoder
         end
 
-        def pack_events(encoded_events)
-          packer = MessagePack::Packer.new
-
+        def write_payload_header(packer)
           packer.write_map_header(3) # Set header with how many elements in the map
 
           packer.write("version")
@@ -137,9 +97,6 @@ module Datadog
           packer.write(Datadog::CI::VERSION::STRING)
 
           packer.write("events")
-          packer.write_array_header(encoded_events.size)
-
-          (packer.buffer.to_a + encoded_events).join
         end
       end
     end

data/lib/datadog/ci/transport/api/agentless.rb

@@ -10,10 +10,11 @@ module Datadog
         class Agentless < Base
           attr_reader :api_key
 
-          def initialize(api_key:, citestcycle_url:, api_url:)
+          def initialize(api_key:, citestcycle_url:, api_url:, citestcov_url:)
             @api_key = api_key
             @citestcycle_http = build_http_client(citestcycle_url, compress: true)
             @api_http = build_http_client(api_url, compress: false)
+            @citestcov_http = build_http_client(citestcov_url, compress: true)
           end
 
           def citestcycle_request(path:, payload:, headers: {}, verb: "post")
@@ -28,6 +29,12 @@ module Datadog
             perform_request(@api_http, path: path, payload: payload, headers: headers, verb: verb)
           end
 
+          def citestcov_request(path:, payload:, headers: {}, verb: "post")
+            super(path: path, payload: payload, headers: headers, verb: verb)
+
+            perform_request(@citestcov_http, path: path, payload: @citestcov_payload, headers: headers, verb: verb)
+          end
+
           private
 
           def perform_request(http_client, path:, payload:, headers:, verb:)

data/lib/datadog/ci/transport/api/base.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "securerandom"
+
 require_relative "../../ext/transport"
 
 module Datadog
@@ -15,6 +17,27 @@ module Datadog
             headers[Ext::Transport::HEADER_CONTENT_TYPE] ||= Ext::Transport::CONTENT_TYPE_MESSAGEPACK
           end
 
+          def citestcov_request(path:, payload:, headers: {}, verb: "post")
+            citestcov_request_boundary = ::SecureRandom.uuid
+
+            headers[Ext::Transport::HEADER_CONTENT_TYPE] ||=
+              "#{Ext::Transport::CONTENT_TYPE_MULTIPART_FORM_DATA}; boundary=#{citestcov_request_boundary}"
+
+            @citestcov_payload = [
+              "--#{citestcov_request_boundary}",
+              'Content-Disposition: form-data; name="event"; filename="event.json"',
+              "Content-Type: application/json",
+              "",
+              '{"dummy":true}',
+              "--#{citestcov_request_boundary}",
+              'Content-Disposition: form-data; name="coverage1"; filename="coverage1.msgpack"',
+              "Content-Type: application/msgpack",
+              "",
+              payload,
+              "--#{citestcov_request_boundary}--"
+            ].join("\r\n")
+          end
+
           def headers_with_default(headers)
             request_headers = default_headers
             request_headers.merge!(headers)

data/lib/datadog/ci/transport/api/builder.rb

@@ -24,15 +24,24 @@ module Datadog
             api_url = settings.ci.agentless_url ||
               "https://#{Ext::Transport::DD_API_HOST_PREFIX}.#{dd_site}:443"
 
-            Agentless.new(api_key: settings.api_key, citestcycle_url: citestcycle_url, api_url: api_url)
+            citestcov_url = settings.ci.agentless_url ||
+              "https://#{Ext::Transport::TEST_COVERAGE_INTAKE_HOST_PREFIX}.#{dd_site}:443"
+
+            Agentless.new(
+              api_key: settings.api_key,
+              citestcycle_url: citestcycle_url,
+              api_url: api_url,
+              citestcov_url: citestcov_url
+            )
           end
 
           def self.build_evp_proxy_api(settings)
             agent_settings = Datadog::Core::Configuration::AgentSettingsResolver.call(settings)
-            negotiation = Datadog::Core::Remote::Negotiation.new(settings, agent_settings)
-
-            # temporary, remove this when patch will be accepted in Core to make logging configurable
-            negotiation.instance_variable_set(:@logged, {no_config_endpoint: true})
+            negotiation = Datadog::Core::Remote::Negotiation.new(
+              settings,
+              agent_settings,
+              suppress_logging: {no_config_endpoint: true}
+            )
 
             evp_proxy_path_prefix = Ext::Transport::EVP_PROXY_PATH_PREFIXES.find do |path_prefix|
               negotiation.endpoint?(path_prefix)

data/lib/datadog/ci/transport/api/evp_proxy.rb

@@ -38,6 +38,14 @@ module Datadog
             perform_request(@agent_api_http, path: path, payload: payload, headers: headers, verb: verb)
           end
 
+          def citestcov_request(path:, payload:, headers: {}, verb: "post")
+            super(path: path, payload: payload, headers: headers, verb: verb)
+
+            headers[Ext::Transport::HEADER_EVP_SUBDOMAIN] = Ext::Transport::TEST_COVERAGE_INTAKE_HOST_PREFIX
+
+            perform_request(@agent_intake_http, path: path, payload: @citestcov_payload, headers: headers, verb: verb)
+          end
+
           private
 
           def perform_request(http_client, path:, payload:, headers:, verb:)

data/lib/datadog/ci/transport/event_platform_transport.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "msgpack"
+
+require "datadog/core/encoding"
+require "datadog/core/chunker"
+
+module Datadog
+  module CI
+    module Transport
+      class EventPlatformTransport
+        DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
+
+        attr_reader :api,
+          :max_payload_size
+
+        def initialize(api:, max_payload_size: DEFAULT_MAX_PAYLOAD_SIZE)
+          @api = api
+          @max_payload_size = max_payload_size
+        end
+
+        def send_events(events)
+          return [] if events.nil? || events.empty?
+
+          Datadog.logger.debug { "[#{self.class.name}] Sending #{events.count} events..." }
+
+          encoded_events = encode_events(events)
+          if encoded_events.empty?
+            Datadog.logger.debug { "[#{self.class.name}] Empty encoded events list, skipping send" }
+            return []
+          end
+
+          responses = []
+
+          Datadog::Core::Chunker.chunk_by_size(encoded_events, max_payload_size).map do |chunk|
+            encoded_payload = pack_events(chunk)
+            Datadog.logger.debug do
+              "[#{self.class.name}] Send chunk of #{chunk.count} events; payload size #{encoded_payload.size}"
+            end
+
+            response = send_payload(encoded_payload)
+
+            responses << response
+          end
+
+          responses
+        end
+
+        private
+
+        def encoder
+          Datadog::Core::Encoding::MsgpackEncoder
+        end
+
+        def pack_events(encoded_events)
+          packer = MessagePack::Packer.new
+
+          write_payload_header(packer)
+
+          packer.write_array_header(encoded_events.count)
+          (packer.buffer.to_a + encoded_events).join
+        end
+
+        def event_too_large?(event, encoded_event)
+          return false unless encoded_event.size > max_payload_size
+
+          # This single event is too large, we can't flush it
+          Datadog.logger.warn("[#{self.class.name}] Dropping coverage event. Payload too large: '#{event.inspect}'")
+          Datadog.logger.warn(encoded_event)
+
+          true
+        end
+
+        def send_payload(encoded_payload)
+          raise NotImplementedError
+        end
+
+        def encode_events(events)
+          raise NotImplementedError
+        end
+
+        def write_payload_header(packer)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end