datadog-ci 0.1.1 → 0.3.0

data/lib/datadog/ci/test_visibility/serializers/span.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Datadog
+  module CI
+    module TestVisibility
+      module Serializers
+        class Span < Base
+          CONTENT_FIELDS = [
+            "trace_id", "span_id", "parent_id",
+            "name", "resource", "service",
+            "error", "start", "duration",
+            "meta", "metrics",
+            "type" => "span_type"
+          ].freeze
+
+          CONTENT_MAP_SIZE = calculate_content_map_size(CONTENT_FIELDS)
+
+          REQUIRED_FIELDS = [
+            "trace_id",
+            "span_id",
+            "error",
+            "name",
+            "resource",
+            "start",
+            "duration"
+          ].freeze
+
+          def content_fields
+            CONTENT_FIELDS
+          end
+
+          def content_map_size
+            CONTENT_MAP_SIZE
+          end
+
+          def type
+            "span"
+          end
+
+          private
+
+          def required_fields
+            REQUIRED_FIELDS
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_visibility/serializers/test_v1.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../../ext/test"
+
+module Datadog
+  module CI
+    module TestVisibility
+      module Serializers
+        class TestV1 < Base
+          CONTENT_FIELDS = [
+            "trace_id", "span_id",
+            "name", "resource", "service",
+            "error", "start", "duration",
+            "meta", "metrics",
+            "type" => "span_type"
+          ].freeze
+
+          CONTENT_MAP_SIZE = calculate_content_map_size(CONTENT_FIELDS)
+
+          REQUIRED_FIELDS = [
+            "trace_id",
+            "span_id",
+            "error",
+            "name",
+            "resource",
+            "start",
+            "duration"
+          ].freeze
+
+          def content_fields
+            CONTENT_FIELDS
+          end
+
+          def content_map_size
+            CONTENT_MAP_SIZE
+          end
+
+          def type
+            "test"
+          end
+
+          def name
+            "#{@span.get_tag(Ext::Test::TAG_FRAMEWORK)}.test"
+          end
+
+          def resource
+            "#{@span.get_tag(Ext::Test::TAG_SUITE)}.#{@span.get_tag(Ext::Test::TAG_NAME)}"
+          end
+
+          private
+
+          def required_fields
+            REQUIRED_FIELDS
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_visibility/transport.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require "msgpack"
+require "uri"
+
+require "datadog/core/encoding"
+require "datadog/core/environment/identity"
+require "datadog/core/chunker"
+
+require_relative "serializers/factories/test_level"
+require_relative "../ext/transport"
+
+module Datadog
+  module CI
+    module TestVisibility
+      class Transport
+        # CI test cycle intake's limit is 5.1MB uncompressed
+        # We will use a bit more conservative value 5MB
+        DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
+
+        attr_reader :serializers_factory,
+          :api,
+          :max_payload_size,
+          :dd_env
+
+        def initialize(
+          api:,
+          dd_env: nil,
+          serializers_factory: Datadog::CI::TestVisibility::Serializers::Factories::TestLevel,
+          max_payload_size: DEFAULT_MAX_PAYLOAD_SIZE
+        )
+          @serializers_factory = serializers_factory
+          @max_payload_size = max_payload_size
+          @dd_env = dd_env
+          @api = api
+        end
+
+        def send_traces(traces)
+          return [] if traces.nil? || traces.empty?
+
+          Datadog.logger.debug { "Sending #{traces.count} traces..." }
+
+          encoded_events = encode_traces(traces)
+          if encoded_events.empty?
+            Datadog.logger.debug { "Empty encoded events list, skipping send" }
+            return []
+          end
+
+          responses = []
+          Datadog::Core::Chunker.chunk_by_size(encoded_events, max_payload_size).map do |chunk|
+            encoded_payload = pack_events(chunk)
+            Datadog.logger.debug do
+              "Send chunk of #{chunk.count} events; payload size #{encoded_payload.size}"
+            end
+
+            response = send_payload(encoded_payload)
+
+            Datadog.logger.debug do
+              "Received server response: #{response.inspect}"
+            end
+
+            responses << response
+          end
+
+          responses
+        end
+
+        private
+
+        def send_payload(encoded_payload)
+          api.request(
+            path: Datadog::CI::Ext::Transport::TEST_VISIBILITY_INTAKE_PATH,
+            payload: encoded_payload
+          )
+        end
+
+        def encode_traces(traces)
+          traces.flat_map do |trace|
+            spans = trace.spans
+            # TODO: remove condition when 1.0 is released
+            if spans.respond_to?(:filter_map)
+              spans.filter_map { |span| encode_span(trace, span) }
+            else
+              trace.spans.map { |span| encode_span(trace, span) }.reject(&:nil?)
+            end
+          end
+        end
+
+        def encode_span(trace, span)
+          serializer = serializers_factory.serializer(trace, span)
+
+          if serializer.valid?
+            encoded = encoder.encode(serializer)
+
+            if encoded.size > max_payload_size
+              # This single event is too large, we can't flush it
+              Datadog.logger.debug { "Dropping test event. Payload too large: '#{span.inspect}'" }
+              Datadog.logger.debug { encoded }
+
+              return nil
+            end
+
+            encoded
+          else
+            Datadog.logger.debug { "Invalid span skipped: #{span}" }
+            nil
+          end
+        end
+
+        def encoder
+          Datadog::Core::Encoding::MsgpackEncoder
+        end
+
+        def pack_events(encoded_events)
+          packer = MessagePack::Packer.new
+
+          packer.write_map_header(3) # Set header with how many elements in the map
+
+          packer.write("version")
+          packer.write(1)
+
+          packer.write("metadata")
+          packer.write_map_header(1)
+
+          packer.write("*")
+          metadata_fields_count = dd_env ? 4 : 3
+          packer.write_map_header(metadata_fields_count)
+
+          if dd_env
+            packer.write("env")
+            packer.write(dd_env)
+          end
+
+          packer.write("runtime-id")
+          packer.write(Datadog::Core::Environment::Identity.id)
+
+          packer.write("language")
+          packer.write(Datadog::Core::Environment::Identity.lang)
+
+          packer.write("library_version")
+          packer.write(Datadog::CI::VERSION::STRING)
+
+          packer.write("events")
+          packer.write_array_header(encoded_events.size)
+
+          (packer.buffer.to_a + encoded_events).join
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/transport/api/base.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "../../ext/transport"
+
+module Datadog
+  module CI
+    module Transport
+      module Api
+        class Base
+          attr_reader :http
+
+          def initialize(http:)
+            @http = http
+          end
+
+          def request(path:, payload:, verb: "post")
+            http.request(
+              path: path,
+              payload: payload,
+              verb: verb,
+              headers: headers
+            )
+          end
+
+          private
+
+          def headers
+            {
+              Ext::Transport::HEADER_CONTENT_TYPE => Ext::Transport::CONTENT_TYPE_MESSAGEPACK
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/transport/api/builder.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "ci_test_cycle"
+require_relative "evp_proxy"
+require_relative "../http"
+require_relative "../../ext/transport"
+
+module Datadog
+  module CI
+    module Transport
+      module Api
+        module Builder
+          def self.build_ci_test_cycle_api(settings)
+            dd_site = settings.site || Ext::Transport::DEFAULT_DD_SITE
+            url = settings.ci.agentless_url ||
+              "https://#{Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX}.#{dd_site}:443"
+
+            uri = URI.parse(url)
+            raise "Invalid agentless mode URL: #{url}" if uri.host.nil?
+
+            http = Datadog::CI::Transport::HTTP.new(
+              host: uri.host,
+              port: uri.port,
+              ssl: uri.scheme == "https" || uri.port == 443,
+              compress: true
+            )
+
+            CiTestCycle.new(api_key: settings.api_key, http: http)
+          end
+
+          def self.build_evp_proxy_api(agent_settings)
+            http = Datadog::CI::Transport::HTTP.new(
+              host: agent_settings.hostname,
+              port: agent_settings.port,
+              ssl: agent_settings.ssl,
+              timeout: agent_settings.timeout_seconds,
+              compress: false
+            )
+
+            EvpProxy.new(http: http)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/transport/api/ci_test_cycle.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../../ext/transport"
+
+module Datadog
+  module CI
+    module Transport
+      module Api
+        class CiTestCycle < Base
+          attr_reader :api_key
+
+          def initialize(api_key:, http:)
+            @api_key = api_key
+
+            super(http: http)
+          end
+
+          private
+
+          def headers
+            headers = super
+            headers[Ext::Transport::HEADER_DD_API_KEY] = api_key
+            headers
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/transport/api/evp_proxy.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "datadog/core/environment/container"
+
+require_relative "base"
+require_relative "../../ext/transport"
+
+module Datadog
+  module CI
+    module Transport
+      module Api
+        class EvpProxy < Base
+          def request(path:, payload:, verb: "post")
+            path = "#{Ext::Transport::EVP_PROXY_PATH_PREFIX}#{path.sub(/^\//, "")}"
+
+            super(
+              path: path,
+              payload: payload,
+              verb: verb
+            )
+          end
+
+          private
+
+          def container_id
+            return @container_id if defined?(@container_id)
+
+            @container_id = Datadog::Core::Environment::Container.container_id
+          end
+
+          def headers
+            headers = super
+            headers[Ext::Transport::HEADER_EVP_SUBDOMAIN] = Ext::Transport::TEST_VISIBILITY_INTAKE_HOST_PREFIX
+
+            c_id = container_id
+            headers[Ext::Transport::HEADER_CONTAINER_ID] = c_id unless c_id.nil?
+
+            headers
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/transport/gzip.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "zlib"
+require "stringio"
+
+module Datadog
+  module CI
+    module Transport
+      module Gzip
+        module_function
+
+        def compress(input)
+          sio = StringIO.new
+          gzip_writer = Zlib::GzipWriter.new(sio, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
+          gzip_writer << input
+          gzip_writer.close
+          sio.string
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/transport/http.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "delegate"
+require "datadog/core/transport/http/adapters/net"
+require "datadog/core/transport/http/env"
+require "datadog/core/transport/request"
+
+require_relative "gzip"
+require_relative "../ext/transport"
+
+module Datadog
+  module CI
+    module Transport
+      class HTTP
+        attr_reader \
+          :host,
+          :port,
+          :ssl,
+          :timeout,
+          :compress
+
+        DEFAULT_TIMEOUT = 30
+
+        def initialize(host:, timeout: DEFAULT_TIMEOUT, port: nil, ssl: true, compress: false)
+          @host = host
+          @port = port
+          @timeout = timeout
+          @ssl = ssl.nil? ? true : ssl
+          @compress = compress.nil? ? false : compress
+        end
+
+        def request(path:, payload:, headers:, verb: "post")
+          if compress
+            headers[Ext::Transport::HEADER_CONTENT_ENCODING] = Ext::Transport::CONTENT_ENCODING_GZIP
+            payload = Gzip.compress(payload)
+          end
+
+          Datadog.logger.debug do
+            "Sending #{verb} request: host=#{host}; port=#{port}; ssl_enabled=#{ssl}; " \
+              "compression_enabled=#{compress}; path=#{path}; payload_size=#{payload.size}"
+          end
+
+          ResponseDecorator.new(
+            adapter.call(
+              build_env(path: path, payload: payload, headers: headers, verb: verb)
+            )
+          )
+        end
+
+        private
+
+        def build_env(path:, payload:, headers:, verb:)
+          env = Datadog::Core::Transport::HTTP::Env.new(
+            Datadog::Core::Transport::Request.new
+          )
+          env.body = payload
+          env.path = path
+          env.headers = headers
+          env.verb = verb
+          env
+        end
+
+        def adapter
+          @adapter ||= Datadog::Core::Transport::HTTP::Adapters::Net.new(host, port, timeout: timeout, ssl: ssl)
+        end
+
+        # this is needed because Datadog::Tracing::Writer is not fully compatiple with Datadog::Core::Transport
+        # TODO: remove before 1.0 when CI implements its own worker
+        class ResponseDecorator < ::SimpleDelegator
+          def trace_count
+            0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/version.rb

@@ -4,8 +4,8 @@ module Datadog
   module CI
     module VERSION
       MAJOR = "0"
-      MINOR = "1"
-      PATCH = "1"
+      MINOR = "3"
+      PATCH = "0"
       PRE = nil
       BUILD = nil
       # PRE and BUILD above are modified for dev gems during gem build GHA workflow

data/sig/datadog/ci/configuration/components.rbs

@@ -5,6 +5,10 @@ module Datadog
         def initialize: (untyped settings) -> void
 
         def activate_ci!: (untyped settings) -> untyped
+
+        def build_agentless_transport: (untyped settings) -> Datadog::CI::TestVisibility::Transport?
+        def build_evp_proxy_transport: (untyped settings, untyped agent_settings) -> Datadog::CI::TestVisibility::Transport
+        def can_use_evp_proxy?: (untyped settings, untyped agent_settings) -> bool
       end
     end
   end

data/sig/datadog/ci/configuration/settings.rbs

@@ -3,6 +3,8 @@ module Datadog
     module Configuration
       module Settings
         extend Datadog::Core::Configuration::Options::ClassMethods
+        include Datadog::Core::Configuration::Options::InstanceMethods
+
         extend Datadog::Core::Configuration::Base::ClassMethods
 
         def self.extended: (untyped base) -> untyped

data/sig/datadog/ci/contrib/cucumber/configuration/settings.rbs

@@ -3,7 +3,7 @@ module Datadog
     module Contrib
       module Cucumber
         module Configuration
-          class Settings < Datadog::Tracing::Contrib::Configuration::Settings
+          class Settings < Datadog::CI::Contrib::Settings
           end
         end
       end

data/sig/datadog/ci/contrib/cucumber/integration.rbs

@@ -3,7 +3,8 @@ module Datadog
     module Contrib
       module Cucumber
         class Integration
-          extend Datadog::Tracing::Contrib::Integration
+          extend Datadog::CI::Contrib::Integration::ClassMethods
+          include Datadog::CI::Contrib::Integration::InstanceMethods
 
           MINIMUM_VERSION: Gem::Version
 

data/sig/datadog/ci/contrib/integration.rbs

@@ -0,0 +1,44 @@
+module Datadog
+  module CI
+    module Contrib
+      module Integration
+        self.@registry: Hash[Symbol, untyped]
+
+        def self.included: (Module base) -> void
+
+        def self.register: (untyped integration, Symbol name) -> void
+
+        def self.registry: () -> Hash[Symbol, Struct[untyped]]
+
+        module ClassMethods
+          def register_as: (Symbol name) -> void
+
+          def version: () -> Gem::Version?
+
+          def available?: () -> bool
+
+          def loaded?: () -> bool
+
+          def compatible?: () -> bool
+
+          def patchable?: () -> bool
+        end
+
+        module InstanceMethods
+          extend ClassMethods
+          @configuration: Datadog::CI::Contrib::Settings?
+
+          def configuration: () -> Datadog::CI::Contrib::Settings
+
+          def configure: (?::Hash[Symbol, untyped] options) ?{ (Datadog::CI::Contrib::Settings) -> Datadog::CI::Contrib::Settings } -> Datadog::CI::Contrib::Settings
+
+          def reset_configuration!: () -> void
+
+          def patcher: () -> Datadog::Tracing::Contrib::Patcher?
+
+          def new_configuration: () -> Datadog::CI::Contrib::Settings
+        end
+      end
+    end
+  end
+end

data/sig/datadog/ci/contrib/minitest/configuration/settings.rbs

@@ -3,7 +3,7 @@ module Datadog
     module Contrib
       module Minitest
         module Configuration
-          class Settings < Datadog::Tracing::Contrib::Configuration::Settings
+          class Settings < Datadog::CI::Contrib::Settings
           end
         end
       end

data/sig/datadog/ci/contrib/minitest/integration.rbs

@@ -3,13 +3,14 @@ module Datadog
     module Contrib
       module Minitest
         class Integration
-          extend Datadog::Tracing::Contrib::Integration
+          extend Datadog::CI::Contrib::Integration::ClassMethods
+          include Datadog::CI::Contrib::Integration::InstanceMethods
 
-          MINIMUM_VERSION: untyped
+          MINIMUM_VERSION: Gem::Version
 
           def self.version: () -> untyped
 
-          def self.loaded?: () -> untyped
+          def self.loaded?: () -> bool
 
           def compatible?: () -> bool
 

data/sig/datadog/ci/contrib/rspec/configuration/settings.rbs

@@ -3,7 +3,7 @@ module Datadog
     module Contrib
       module RSpec
         module Configuration
-          class Settings < Datadog::Tracing::Contrib::Configuration::Settings
+          class Settings < Datadog::CI::Contrib::Settings
           end
         end
       end

data/sig/datadog/ci/contrib/rspec/integration.rbs

@@ -3,9 +3,10 @@ module Datadog
     module Contrib
       module RSpec
         class Integration
-          extend Datadog::Tracing::Contrib::Integration
+          extend Datadog::CI::Contrib::Integration::ClassMethods
+          include Datadog::CI::Contrib::Integration::InstanceMethods
 
-          MINIMUM_VERSION: untyped
+          MINIMUM_VERSION: Gem::Version
 
           def self.version: () -> untyped