datadog-ci 0.1.1 → 0.2.0

data/lib/datadog/ci/contrib/settings.rb

@@ -0,0 +1,33 @@
+require "datadog/core/configuration/base"
+
+module Datadog
+  module CI
+    module Contrib
+      # Common settings for all integrations
+      # @public_api
+      class Settings
+        include Core::Configuration::Base
+
+        option :enabled, default: true
+        option :service_name
+        option :operation_name
+
+        def configure(options = {})
+          self.class.options.each do |name, _value|
+            self[name] = options[name] if options.key?(name)
+          end
+
+          yield(self) if block_given?
+        end
+
+        def [](name)
+          respond_to?(name) ? send(name) : get_option(name)
+        end
+
+        def []=(name, value)
+          respond_to?("#{name}=") ? send("#{name}=", value) : set_option(name, value)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/ext/environment/providers/local_git.rb

@@ -97,6 +97,13 @@ module Datadog
 
               raise "Failed to run git command #{cmd}: #{out}" unless status.success?
 
+              # Sometimes Encoding.default_external is somehow set to US-ASCII which breaks
+              # commit messages with UTF-8 characters like emojis
+              # We force output's encoding to be UTF-8 in this case
+              # This is safe to do as UTF-8 is compatible with US-ASCII
+              if Encoding.default_external == Encoding::US_ASCII
+                out = out.force_encoding(Encoding::UTF_8)
+              end
               out.strip! # There's always a "\n" at the end of the command output
 
               return nil if out.empty?

data/lib/datadog/ci/ext/settings.rb

@@ -6,6 +6,8 @@ module Datadog
       # Defines constants for test tags
       module Settings
         ENV_MODE_ENABLED = "DD_TRACE_CI_ENABLED"
+        ENV_AGENTLESS_MODE_ENABLED = "DD_CIVISIBILITY_AGENTLESS_ENABLED"
+        ENV_AGENTLESS_URL = "DD_CIVISIBILITY_AGENTLESS_URL"
       end
     end
   end

data/lib/datadog/ci/ext/transport.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Ext
+      module Transport
+        HEADER_DD_API_KEY = "DD-API-KEY"
+        HEADER_CONTENT_TYPE = "Content-Type"
+        HEADER_CONTENT_ENCODING = "Content-Encoding"
+
+        TEST_VISIBILITY_INTAKE_HOST_PREFIX = "citestcycle-intake"
+        TEST_VISIBILITY_INTAKE_PATH = "/api/v2/citestcycle"
+
+        CONTENT_TYPE_MESSAGEPACK = "application/msgpack"
+        CONTENT_ENCODING_GZIP = "gzip"
+      end
+    end
+  end
+end

data/lib/datadog/ci/{test.rb → recorder.rb}

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "datadog/tracing"
 require "datadog/tracing/contrib/analytics"
 
 require_relative "ext/app_types"
@@ -11,7 +12,7 @@ require "rbconfig"
 module Datadog
   module CI
     # Common behavior for CI tests
-    module Test
+    module Recorder
       # Creates a new span for a CI test
       def self.trace(span_name, options = {})
         span_options = {
@@ -19,13 +20,13 @@ module Datadog
         }.merge(options[:span_options] || {})
 
         if block_given?
-          Tracing.trace(span_name, **span_options) do |span, trace|
+          ::Datadog::Tracing.trace(span_name, **span_options) do |span, trace|
             set_tags!(trace, span, options)
             yield(span, trace)
           end
         else
-          span = Tracing.trace(span_name, **span_options)
-          trace = Tracing.active_trace
+          span = ::Datadog::Tracing.trace(span_name, **span_options)
+          trace = ::Datadog::Tracing.active_trace
           set_tags!(trace, span, options)
           span
         end
@@ -37,7 +38,7 @@ module Datadog
 
         # Set default tags
         trace.origin = Ext::Test::CONTEXT_ORIGIN if trace
-        Datadog::Tracing::Contrib::Analytics.set_measured(span)
+        ::Datadog::Tracing::Contrib::Analytics.set_measured(span)
         span.set_tag(Ext::Test::TAG_SPAN_KIND, Ext::AppTypes::TYPE_TEST)
 
         # Set environment tags

data/lib/datadog/ci/test_visibility/flush.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "datadog/tracing/metadata/ext"
+require "datadog/tracing/flush"
+
+module Datadog
+  module CI
+    module TestVisibility
+      module Flush
+        # Common behavior for CI flushing
+        module Tagging
+          # Decorate a trace with CI tags
+          def get_trace(trace_op)
+            trace = trace_op.flush!
+
+            # Origin tag is required on every span
+            trace.spans.each do |span|
+              span.set_tag(
+                Tracing::Metadata::Ext::Distributed::TAG_ORIGIN,
+                trace.origin
+              )
+            end
+
+            trace
+          end
+        end
+
+        # Consumes only completed traces (where all spans have finished)
+        class Finished < Tracing::Flush::Finished
+          prepend Tagging
+        end
+
+        # Performs partial trace flushing to avoid large traces residing in memory for too long
+        class Partial < Tracing::Flush::Partial
+          prepend Tagging
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_visibility/serializers/base.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module TestVisibility
+      module Serializers
+        class Base
+          MINIMUM_TIMESTAMP_NANO = 946684800000000000
+          MINIMUM_DURATION_NANO = 0
+          MAXIMUM_DURATION_NANO = 9223372036854775807
+
+          attr_reader :trace, :span
+
+          def initialize(trace, span)
+            @trace = trace
+            @span = span
+          end
+
+          def to_msgpack(packer = nil)
+            packer ||= MessagePack::Packer.new
+
+            packer.write_map_header(3)
+
+            write_field(packer, "type")
+            write_field(packer, "version")
+
+            packer.write("content")
+            packer.write_map_header(content_map_size)
+
+            content_fields.each do |field|
+              if field.is_a?(Hash)
+                field.each do |field_name, method|
+                  write_field(packer, field_name, method)
+                end
+              else
+                write_field(packer, field)
+              end
+            end
+          end
+
+          # validates according to citestcycle json schema
+          def valid?
+            required_fields_present? && valid_start_time? && valid_duration?
+          end
+
+          def content_fields
+            []
+          end
+
+          def content_map_size
+            0
+          end
+
+          def runtime_id
+            @trace.runtime_id
+          end
+
+          def trace_id
+            @trace.id
+          end
+
+          def span_id
+            @span.id
+          end
+
+          def parent_id
+            @span.parent_id
+          end
+
+          def type
+          end
+
+          def version
+            1
+          end
+
+          def span_type
+            @span.type
+          end
+
+          def name
+            @span.name
+          end
+
+          def resource
+            @span.resource
+          end
+
+          def service
+            @span.service
+          end
+
+          def start
+            @start ||= time_nano(@span.start_time)
+          end
+
+          def duration
+            @duration ||= duration_nano(@span.duration)
+          end
+
+          def meta
+            @span.meta
+          end
+
+          def metrics
+            @span.metrics
+          end
+
+          def error
+            @span.status
+          end
+
+          def self.calculate_content_map_size(fields_list)
+            fields_list.reduce(0) do |size, field|
+              if field.is_a?(Hash)
+                size + field.size
+              else
+                size + 1
+              end
+            end
+          end
+
+          private
+
+          def valid_start_time?
+            !start.nil? && start >= MINIMUM_TIMESTAMP_NANO
+          end
+
+          def valid_duration?
+            !duration.nil? && duration >= MINIMUM_DURATION_NANO && duration <= MAXIMUM_DURATION_NANO
+          end
+
+          def required_fields_present?
+            required_fields.all? { |field| !send(field).nil? }
+          end
+
+          def required_fields
+            []
+          end
+
+          def write_field(packer, field_name, method = nil)
+            method ||= field_name
+
+            packer.write(field_name)
+            packer.write(send(method))
+          end
+
+          # in nanoseconds since Epoch
+          def time_nano(time)
+            time.to_i * 1000000000 + time.nsec
+          end
+
+          # in nanoseconds
+          def duration_nano(duration)
+            (duration * 1e9).to_i
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_visibility/serializers/factories/test_level.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative "../test_v1"
+require_relative "../span"
+
+module Datadog
+  module CI
+    module TestVisibility
+      module Serializers
+        module Factories
+          # This factory takes care of creating citestcycle serializers when test-level visibility is enabled
+          # NOTE: citestcycle is a protocol Datadog uses to submit test execution tracing information to CI visibility
+          # backend
+          module TestLevel
+            module_function
+
+            def serializer(trace, span)
+              case span.type
+              when Datadog::CI::Ext::AppTypes::TYPE_TEST
+                Serializers::TestV1.new(trace, span)
+              else
+                Serializers::Span.new(trace, span)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_visibility/serializers/span.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Datadog
+  module CI
+    module TestVisibility
+      module Serializers
+        class Span < Base
+          CONTENT_FIELDS = [
+            "trace_id", "span_id", "parent_id",
+            "name", "resource", "service",
+            "error", "start", "duration",
+            "meta", "metrics",
+            "type" => "span_type"
+          ].freeze
+
+          CONTENT_MAP_SIZE = calculate_content_map_size(CONTENT_FIELDS)
+
+          REQUIRED_FIELDS = [
+            "trace_id",
+            "span_id",
+            "error",
+            "name",
+            "resource",
+            "start",
+            "duration"
+          ].freeze
+
+          def content_fields
+            CONTENT_FIELDS
+          end
+
+          def content_map_size
+            CONTENT_MAP_SIZE
+          end
+
+          def type
+            "span"
+          end
+
+          private
+
+          def required_fields
+            REQUIRED_FIELDS
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_visibility/serializers/test_v1.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../../ext/test"
+
+module Datadog
+  module CI
+    module TestVisibility
+      module Serializers
+        class TestV1 < Base
+          CONTENT_FIELDS = [
+            "trace_id", "span_id",
+            "name", "resource", "service",
+            "error", "start", "duration",
+            "meta", "metrics",
+            "type" => "span_type"
+          ].freeze
+
+          CONTENT_MAP_SIZE = calculate_content_map_size(CONTENT_FIELDS)
+
+          REQUIRED_FIELDS = [
+            "trace_id",
+            "span_id",
+            "error",
+            "name",
+            "resource",
+            "start",
+            "duration"
+          ].freeze
+
+          def content_fields
+            CONTENT_FIELDS
+          end
+
+          def content_map_size
+            CONTENT_MAP_SIZE
+          end
+
+          def type
+            "test"
+          end
+
+          def name
+            "#{@span.get_tag(Ext::Test::TAG_FRAMEWORK)}.test"
+          end
+
+          def resource
+            "#{@span.get_tag(Ext::Test::TAG_SUITE)}.#{@span.get_tag(Ext::Test::TAG_NAME)}"
+          end
+
+          private
+
+          def required_fields
+            REQUIRED_FIELDS
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/test_visibility/transport.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require "msgpack"
+require "uri"
+
+require "datadog/core/encoding"
+require "datadog/core/environment/identity"
+require "datadog/core/chunker"
+
+require_relative "serializers/factories/test_level"
+require_relative "../ext/transport"
+require_relative "../transport/http"
+
+module Datadog
+  module CI
+    module TestVisibility
+      class Transport
+        # CI test cycle intake's limit is 5.1MB uncompressed
+        # We will use a bit more conservative value 5MB
+        DEFAULT_MAX_PAYLOAD_SIZE = 5 * 1024 * 1024
+
+        attr_reader :serializers_factory,
+          :api_key,
+          :max_payload_size,
+          :http,
+          :env
+
+        def initialize(
+          api_key:,
+          url:,
+          env: nil,
+          serializers_factory: Datadog::CI::TestVisibility::Serializers::Factories::TestLevel,
+          max_payload_size: DEFAULT_MAX_PAYLOAD_SIZE
+        )
+          @serializers_factory = serializers_factory
+          @api_key = api_key
+          @max_payload_size = max_payload_size
+          @env = env
+
+          uri = URI.parse(url)
+
+          raise "Invalid agentless mode URL: #{url}" if uri.host.nil?
+
+          @http = Datadog::CI::Transport::HTTP.new(
+            host: uri.host,
+            port: uri.port,
+            ssl: uri.scheme == "https" || uri.port == 443,
+            compress: true
+          )
+        end
+
+        def send_traces(traces)
+          return [] if traces.nil? || traces.empty?
+
+          Datadog.logger.debug { "Sending #{traces.count} traces..." }
+
+          encoded_events = encode_traces(traces)
+          if encoded_events.empty?
+            Datadog.logger.debug { "Empty encoded events list, skipping send" }
+            return []
+          end
+
+          responses = []
+          Datadog::Core::Chunker.chunk_by_size(encoded_events, max_payload_size).map do |chunk|
+            encoded_payload = pack_events(chunk)
+            Datadog.logger.debug do
+              "Send chunk of #{chunk.count} events; payload size #{encoded_payload.size}"
+            end
+
+            response = send_payload(encoded_payload)
+
+            Datadog.logger.debug do
+              "Received server response: #{response.inspect}"
+            end
+
+            responses << response
+          end
+
+          responses
+        end
+
+        private
+
+        def send_payload(encoded_payload)
+          http.request(
+            path: Datadog::CI::Ext::Transport::TEST_VISIBILITY_INTAKE_PATH,
+            payload: encoded_payload,
+            headers: {
+              Ext::Transport::HEADER_DD_API_KEY => api_key,
+              Ext::Transport::HEADER_CONTENT_TYPE => Ext::Transport::CONTENT_TYPE_MESSAGEPACK
+            }
+          )
+        end
+
+        def encode_traces(traces)
+          traces.flat_map do |trace|
+            spans = trace.spans
+            # TODO: remove condition when 1.0 is released
+            if spans.respond_to?(:filter_map)
+              spans.filter_map { |span| encode_span(trace, span) }
+            else
+              trace.spans.map { |span| encode_span(trace, span) }.reject(&:nil?)
+            end
+          end
+        end
+
+        def encode_span(trace, span)
+          serializer = serializers_factory.serializer(trace, span)
+
+          if serializer.valid?
+            encoded = encoder.encode(serializer)
+
+            if encoded.size > max_payload_size
+              # This single event is too large, we can't flush it
+              Datadog.logger.debug { "Dropping test event. Payload too large: '#{span.inspect}'" }
+              Datadog.logger.debug { encoded }
+
+              return nil
+            end
+
+            encoded
+          else
+            Datadog.logger.debug { "Invalid span skipped: #{span}" }
+            nil
+          end
+        end
+
+        def encoder
+          Datadog::Core::Encoding::MsgpackEncoder
+        end
+
+        def pack_events(encoded_events)
+          packer = MessagePack::Packer.new
+
+          packer.write_map_header(3) # Set header with how many elements in the map
+
+          packer.write("version")
+          packer.write(1)
+
+          packer.write("metadata")
+          packer.write_map_header(1)
+
+          packer.write("*")
+          metadata_fields_count = env ? 4 : 3
+          packer.write_map_header(metadata_fields_count)
+
+          if env
+            packer.write("env")
+            packer.write(env)
+          end
+
+          packer.write("runtime-id")
+          packer.write(Datadog::Core::Environment::Identity.id)
+
+          packer.write("language")
+          packer.write(Datadog::Core::Environment::Identity.lang)
+
+          packer.write("library_version")
+          packer.write(Datadog::CI::VERSION::STRING)
+
+          packer.write("events")
+          packer.write_array_header(encoded_events.size)
+
+          (packer.buffer.to_a + encoded_events).join
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/transport/gzip.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "zlib"
+require "stringio"
+
+module Datadog
+  module CI
+    module Transport
+      module Gzip
+        module_function
+
+        def compress(input)
+          gzip_writer = Zlib::GzipWriter.new(StringIO.new)
+          gzip_writer << input
+          gzip_writer.close.string
+        end
+      end
+    end
+  end
+end