datadog 2.15.0 → 2.17.0

data/lib/datadog/core/telemetry/transport/http/api.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative '../../../encoding'
+require_relative '../../../transport/http/api/map'
+require_relative '../../../transport/http/api/instance'
+require_relative '../../../transport/http/api/spec'
+require_relative 'telemetry'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Transport
+        module HTTP
+          # Namespace for API components
+          module API
+            # Default API versions
+            AGENT_TELEMETRY = 'agent_telemetry'
+            AGENTLESS_TELEMETRY = 'agentless_telemetry'
+
+            module_function
+
+            def defaults
+              Datadog::Core::Transport::HTTP::API::Map[
+                AGENT_TELEMETRY => Telemetry::API::Spec.new do |s|
+                  s.telemetry = Telemetry::API::Endpoint.new(
+                    '/telemetry/proxy/api/v2/apmtelemetry',
+                    Core::Encoding::JSONEncoder,
+                  )
+                end,
+                AGENTLESS_TELEMETRY => Telemetry::API::Spec.new do |s|
+                  s.telemetry = Telemetry::API::Endpoint.new(
+                    '/api/v2/apmtelemetry',
+                    Core::Encoding::JSONEncoder,
+                  )
+                end,
+              ]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/transport/http/client.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative '../../../transport/http/env'
+require_relative '../../../transport/http/response'
+
+# TODO: Decouple transport/http/client
+#
+# The standard one does `include Transport::HTTP::Statistics` and performs
+# stats updates, which may or may not be desirable in general.
+
+module Datadog
+  module Core
+    module Telemetry
+      module Transport
+        module HTTP
+          # Routes, encodes, and sends DI data to the trace agent via HTTP.
+          class Client
+            attr_reader :api, :logger
+
+            def initialize(api, logger:)
+              @api = api
+              @logger = logger
+            end
+
+            def send_request(request, &block)
+              # Build request into env
+              env = build_env(request)
+
+              # Get responses from API
+              yield(api, env)
+            rescue => e
+              message =
+                "Internal error during #{self.class.name} request. Cause: #{e.class.name} #{e.message} " \
+                  "Location: #{Array(e.backtrace).first}"
+
+              logger.debug(message)
+
+              Datadog::Core::Transport::InternalErrorResponse.new(e)
+            end
+
+            def build_env(request)
+              Datadog::Core::Transport::HTTP::Env.new(request)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/transport/http/telemetry.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require_relative '../../../transport/http/api/endpoint'
+require_relative '../../../transport/http/api/instance'
+require_relative '../../../transport/http/api/spec'
+require_relative '../../../transport/request'
+require_relative 'client'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Transport
+        module HTTP
+          module Telemetry
+            module Client
+              def send_telemetry_payload(request)
+                send_request(request) do |api, env| # steep:ignore
+                  api.send_telemetry(env)
+                end
+              end
+            end
+
+            module API
+              class Instance < Core::Transport::HTTP::API::Instance
+                def send_telemetry(env)
+                  raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new('telemetry', self) unless spec.is_a?(Telemetry::API::Spec)
+
+                  spec.send_telemetry(env) do |request_env|
+                    call(request_env)
+                  end
+                end
+              end
+
+              class Spec < Core::Transport::HTTP::API::Spec
+                attr_accessor :telemetry
+
+                def send_telemetry(env, &block)
+                  raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('telemetry', self) if telemetry.nil?
+
+                  telemetry.call(env, &block)
+                end
+              end
+
+              class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
+                HEADER_CONTENT_TYPE = 'Content-Type'
+
+                attr_reader \
+                  :encoder
+
+                def initialize(path, encoder)
+                  super(:post, path)
+                  @encoder = encoder
+                end
+
+                def call(env, &block)
+                  # Encode body & type
+                  env.headers[HEADER_CONTENT_TYPE] = encoder.content_type
+                  env.headers.update(headers(
+                    request_type: env.request.request_type,
+                    api_key: env.request.api_key,
+                  ))
+                  env.body = env.request.parcel.data
+
+                  super
+                end
+
+                def headers(request_type:, api_key:, api_version: 'v2')
+                  {
+                    Core::Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST => '1',
+                    # Provided by encoder
+                    # 'Content-Type' => 'application/json',
+                    'DD-Telemetry-API-Version' => api_version,
+                    'DD-Telemetry-Request-Type' => request_type,
+                    'DD-Client-Library-Language' => Core::Environment::Ext::LANG,
+                    'DD-Client-Library-Version' => Core::Environment::Identity.gem_datadog_version_semver2,
+
+                    # Enable debug mode for telemetry
+                    # 'DD-Telemetry-Debug-Enabled' => 'true',
+                  }.tap do |result|
+                    result['DD-API-KEY'] = api_key unless api_key.nil?
+                  end
+                end
+              end
+            end
+          end
+
+          HTTP::Client.include(Telemetry::Client)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/transport/http.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative 'telemetry'
+require_relative 'http/api'
+require_relative '../../transport/http'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Transport
+        # Namespace for HTTP transport components
+        module HTTP
+          module_function
+
+          # Builds a new Transport::HTTP::Client with default settings
+          # Pass a block to override any settings.
+          def agentless_telemetry(
+            agent_settings:,
+            logger:,
+            api_key: nil,
+            api_version: nil,
+            headers: nil
+          )
+            Core::Transport::HTTP.build(api_instance_class: Telemetry::API::Instance,
+              logger: logger,
+              agent_settings: agent_settings,
+              api_version: api_version,
+              headers: headers) do |transport|
+              apis = API.defaults
+
+              transport.api API::AGENTLESS_TELEMETRY, apis[API::AGENTLESS_TELEMETRY]
+
+              # Call block to apply any customization, if provided
+              yield(transport) if block_given?
+            end.to_transport(Core::Telemetry::Transport::Telemetry::Transport).tap do |transport|
+              transport.api_key = api_key
+            end
+          end
+
+          # Builds a new Transport::HTTP::Client with default settings
+          # Pass a block to override any settings.
+          def agent_telemetry(
+            agent_settings:,
+            logger:,
+            api_version: nil,
+            headers: nil
+          )
+            Core::Transport::HTTP.build(api_instance_class: Telemetry::API::Instance,
+              logger: logger,
+              agent_settings: agent_settings, api_version: api_version, headers: headers) do |transport|
+              apis = API.defaults
+
+              transport.api API::AGENT_TELEMETRY, apis[API::AGENT_TELEMETRY]
+
+              # Call block to apply any customization, if provided
+              yield(transport) if block_given?
+            end.to_transport(Core::Telemetry::Transport::Telemetry::Transport)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/transport/telemetry.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative '../../transport/parcel'
+require_relative 'http/client'
+require_relative 'http/telemetry'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Transport
+        module Telemetry
+          class EncodedParcel
+            include Datadog::Core::Transport::Parcel
+          end
+
+          class Request < Datadog::Core::Transport::Request
+            attr_reader :request_type
+            attr_reader :api_key
+
+            def initialize(request_type, parcel, api_key)
+              @request_type = request_type
+              super(parcel)
+              @api_key = api_key
+            end
+          end
+
+          class Transport
+            attr_reader :client, :apis, :default_api, :current_api_id, :logger
+            attr_accessor :api_key
+
+            def initialize(apis, default_api, logger:)
+              @apis = apis
+              @logger = logger
+
+              @client = HTTP::Client.new(@apis[default_api], logger: logger)
+            end
+
+            def send_telemetry(request_type:, payload:)
+              json = JSON.dump(payload)
+              parcel = EncodedParcel.new(json)
+              request = Request.new(request_type, parcel, api_key)
+
+              @client.send_telemetry_payload(request)
+              # Perform no error checking here
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/worker.rb

@@ -17,8 +17,6 @@ module Datadog
         DEFAULT_BUFFER_MAX_SIZE = 1000
         APP_STARTED_EVENT_RETRIES = 10
 
-        TELEMETRY_STARTED_ONCE = Utils::OnlyOnceSuccessful.new(APP_STARTED_EVENT_RETRIES)
-
         def initialize(
           heartbeat_interval_seconds:,
           metrics_aggregation_interval_seconds:,
@@ -48,14 +46,25 @@ module Datadog
           @buffer_size = buffer_size
 
           self.buffer = buffer_klass.new(@buffer_size)
+
+          @initial_event_once = Utils::OnlyOnceSuccessful.new(APP_STARTED_EVENT_RETRIES)
         end
 
         attr_reader :logger
+        attr_reader :initial_event_once
+        attr_reader :initial_event
 
-        def start
+        # Returns true if worker thread is successfully started,
+        # false if worker thread was not started but telemetry is enabled,
+        # nil if telemetry is disabled.
+        def start(initial_event)
           return if !enabled? || forked?
 
+          @initial_event = initial_event
+
           # starts async worker
+          # perform should return true if thread was actually started,
+          # false otherwise
           perform
         end
 
@@ -65,18 +74,60 @@ module Datadog
           super
         end
 
+        # Returns true if event was enqueued, nil if not.
+        # While returning false may seem more reasonable, the only reason
+        # for not enqueueing event (presently) is that telemetry is disabled
+        # altogether, and in this case other methods return nil.
         def enqueue(event)
           return if !enabled? || forked?
 
           buffer.push(event)
+          true
+        end
+
+        def sent_initial_event?
+          initial_event_once.success?
         end
 
-        def sent_started_event?
-          TELEMETRY_STARTED_ONCE.success?
+        def failed_initial_event?
+          initial_event_once.failed?
         end
 
-        def failed_to_start?
-          TELEMETRY_STARTED_ONCE.failed?
+        def need_initial_event?
+          !sent_initial_event? && !failed_initial_event?
+        end
+
+        # Wait for the worker to send out all events that have already
+        # been queued, up to 15 seconds. Returns whether all events have
+        # been flushed.
+        #
+        # @api private
+        def flush
+          return true unless enabled? || !run_loop?
+
+          started = Utils::Time.get_time
+          loop do
+            # The AppStarted event is triggered by the worker itself,
+            # from the worker thread. As such the main thread has no way
+            # to delay itself until that event is queued and we need some
+            # way to wait until that event is sent out to assert on it in
+            # the test suite. Check the run once flag which *should*
+            # indicate the event has been queued (at which point our queue
+            # depth check should waint until it's sent).
+            # This is still a hack because the flag can be overridden
+            # either way with or without the event being sent out.
+            # Note that if the AppStarted sending fails, this check
+            # will return false and flushing will be blocked until the
+            # 15 second timeout.
+            # Note that the first wait interval between telemetry event
+            # sending is 10 seconds, the timeout needs to be strictly
+            # greater than that.
+            return true if buffer.empty? && !in_iteration? && sent_initial_event?
+
+            sleep 0.5
+
+            return false if Utils::Time.get_time - started > 15
+          end
         end
 
         private
@@ -84,11 +135,26 @@ module Datadog
         def perform(*events)
           return if !enabled? || forked?
 
-          started! unless sent_started_event?
+          if need_initial_event?
+            started!
+            unless sent_initial_event?
+              # We still haven't succeeded in sending the started event,
+              # which will make flush_events do nothing - but the events
+              # given to us as the parameter have already been removed
+              # from the queue.
+              # Put the events back to the front of the queue to not
+              # lose them.
+              buffer.unshift(*events)
+              return
+            end
+          end
 
           metric_events = @metrics_manager.flush!
           events = [] if events.nil?
-          flush_events(events + metric_events)
+          events += metric_events
+          if events.any?
+            flush_events(events)
+          end
 
           @current_ticks += 1
           return if @current_ticks < @ticks_per_heartbeat
@@ -98,9 +164,6 @@ module Datadog
         end
 
         def flush_events(events)
-          return if events.empty?
-          return if !enabled? || !sent_started_event?
-
           events = deduplicate_logs(events)
 
           logger.debug { "Sending #{events&.count} telemetry events" }
@@ -108,7 +171,7 @@ module Datadog
         end
 
         def heartbeat!
-          return if !enabled? || !sent_started_event?
+          return if !enabled? || !sent_initial_event?
 
           send_event(Event::AppHeartbeat.new)
         end
@@ -116,26 +179,29 @@ module Datadog
         def started!
           return unless enabled?
 
-          if failed_to_start?
-            logger.debug('Telemetry app-started event exhausted retries, disabling telemetry worker')
-            disable!
-            return
-          end
-
-          TELEMETRY_STARTED_ONCE.run do
-            res = send_event(Event::AppStarted.new)
+          initial_event_once.run do
+            res = send_event(initial_event)
 
             if res.ok?
-              logger.debug('Telemetry app-started event is successfully sent')
+              logger.debug { "Telemetry initial event (#{initial_event.type}) is successfully sent" }
 
-              send_event(Event::AppDependenciesLoaded.new) if @dependency_collection
+              # TODO Dependencies loaded event should probably check for new
+              # dependencies and send the new ones.
+              # System tests demand only one instance of this event per
+              # dependency.
+              send_event(Event::AppDependenciesLoaded.new) if @dependency_collection && initial_event.class.eql?(Telemetry::Event::AppStarted) # standard:disable Style/ClassEqualityComparison:
 
               true
             else
-              logger.debug('Error sending telemetry app-started event, retry after heartbeat interval...')
+              logger.debug("Error sending telemetry initial event (#{initial_event.type}), retry after heartbeat interval...")
               false
             end
           end
+
+          if failed_initial_event?
+            logger.debug { "Telemetry initial event (#{initial_event.type}) exhausted retries, disabling telemetry worker" }
+            disable!
+          end
         end
 
         def send_event(event)

data/lib/datadog/core/transport/http/adapters/test.rb

@@ -38,7 +38,8 @@ module Datadog
               @status = status
             end
 
-            def url; end
+            def url
+            end
 
             # Response for test adapter
             class Response

data/lib/datadog/core/transport/http/builder.rb

@@ -41,19 +41,19 @@ module Datadog
 
           def adapter(config, *args, **kwargs)
             @default_adapter = case config
-                               when Core::Configuration::AgentSettingsResolver::AgentSettings
-                                 registry_klass = REGISTRY.get(config.adapter)
-                                 raise UnknownAdapterError, config.adapter if registry_klass.nil?
-
-                                 registry_klass.build(config)
-                               when Symbol
-                                 registry_klass = REGISTRY.get(config)
-                                 raise UnknownAdapterError, config if registry_klass.nil?
-
-                                 registry_klass.new(*args, **kwargs)
-                               else
-                                 config
-                               end
+            when Core::Configuration::AgentSettingsResolver::AgentSettings
+              registry_klass = REGISTRY.get(config.adapter)
+              raise UnknownAdapterError, config.adapter if registry_klass.nil?
+
+              registry_klass.build(config)
+            when Symbol
+              registry_klass = REGISTRY.get(config)
+              raise UnknownAdapterError, config if registry_klass.nil?
+
+              registry_klass.new(*args, **kwargs)
+            else
+              config
+            end
           end
 
           def headers(values = {})

data/lib/datadog/core/utils/at_fork_monkey_patch.rb

@@ -51,13 +51,13 @@ module Datadog
           def fork
             # If a block is provided, it must be wrapped to trigger callbacks.
             child_block = if block_given?
-                            proc do
-                              AtForkMonkeyPatch.run_at_fork_blocks(:child)
+              proc do
+                AtForkMonkeyPatch.run_at_fork_blocks(:child)
 
-                              # Invoke original block
-                              yield
-                            end
-                          end
+                # Invoke original block
+                yield
+              end
+            end
 
             # Start fork
             # If a block is provided, use the wrapped version.

data/lib/datadog/core/utils/duration.rb

@@ -7,42 +7,42 @@ module Datadog
       module Duration
         def self.call(value, base: :s)
           cast = if value.include?('.')
-                   method(:Float)
-                 else
-                   method(:Integer)
-                 end
+            method(:Float)
+          else
+            method(:Integer)
+          end
 
           scale = case base
-                  when :s
-                    1_000_000_000
-                  when :ms
-                    1_000_000
-                  when :us
-                    1000
-                  when :ns
-                    1
-                  else
-                    raise ArgumentError, "invalid base: #{base.inspect}"
-                  end
+          when :s
+            1_000_000_000
+          when :ms
+            1_000_000
+          when :us
+            1000
+          when :ns
+            1
+          else
+            raise ArgumentError, "invalid base: #{base.inspect}"
+          end
 
           result = case value
-                   when /^(\d+(?:\.\d+)?)h$/
-                     cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 * 60 / scale
-                   when /^(\d+(?:\.\d+)?)m$/
-                     cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 / scale
-                   when /^(\d+(?:\.\d+)?)s$/
-                     cast.call(Regexp.last_match(1)) * 1_000_000_000 / scale
-                   when /^(\d+(?:\.\d+)?)ms$/
-                     cast.call(Regexp.last_match(1)) * 1_000_000 / scale
-                   when /^(\d+(?:\.\d+)?)us$/
-                     cast.call(Regexp.last_match(1)) * 1_000 / scale
-                   when /^(\d+(?:\.\d+)?)ns$/
-                     cast.call(Regexp.last_match(1)) / scale
-                   when /^(\d+(?:\.\d+)?)$/
-                     cast.call(Regexp.last_match(1))
-                   else
-                     raise ArgumentError, "invalid duration: #{value.inspect}"
-                   end
+          when /^(\d+(?:\.\d+)?)h$/
+            cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 * 60 / scale
+          when /^(\d+(?:\.\d+)?)m$/
+            cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 / scale
+          when /^(\d+(?:\.\d+)?)s$/
+            cast.call(Regexp.last_match(1)) * 1_000_000_000 / scale
+          when /^(\d+(?:\.\d+)?)ms$/
+            cast.call(Regexp.last_match(1)) * 1_000_000 / scale
+          when /^(\d+(?:\.\d+)?)us$/
+            cast.call(Regexp.last_match(1)) * 1_000 / scale
+          when /^(\d+(?:\.\d+)?)ns$/
+            cast.call(Regexp.last_match(1)) / scale
+          when /^(\d+(?:\.\d+)?)$/
+            cast.call(Regexp.last_match(1))
+          else
+            raise ArgumentError, "invalid duration: #{value.inspect}"
+          end
           # @type var result: Numeric
           result.round
         end

data/lib/datadog/core/utils/forking.rb

@@ -47,12 +47,12 @@ module Datadog
           # This wrapper prevents this by initializing the fork PID when the object is created.
           if RUBY_VERSION >= '3'
             def initialize(*args, **kwargs, &block)
-              super(*args, **kwargs, &block)
+              super
               update_fork_pid!
             end
           else
             def initialize(*args, &block)
-              super(*args, &block)
+              super
               update_fork_pid!
             end
           end

data/lib/datadog/core/utils/network.rb

@@ -32,7 +32,7 @@ module Datadog
           def stripped_ip_from_request_headers(headers, ip_headers_to_check: DEFAULT_IP_HEADERS_NAMES)
             ip = ip_header(headers, ip_headers_to_check)
 
-            ip ? ip.to_s : nil
+            ip&.to_s
           end
 
           # @param [String] IP value.
@@ -40,7 +40,7 @@ module Datadog
           # @return [nil] when no valid IP value found.
           def stripped_ip(ip)
             ip = ip_to_ipaddr(ip)
-            ip ? ip.to_s : nil
+            ip&.to_s
           end
 
           private
@@ -52,10 +52,10 @@ module Datadog
             return unless ip
 
             clean_ip = if likely_ipv4?(ip)
-                         strip_ipv4_port(ip)
-                       else
-                         strip_zone_specifier(strip_ipv6_port(ip))
-                       end
+              strip_ipv4_port(ip)
+            else
+              strip_zone_specifier(strip_ipv6_port(ip))
+            end
 
             begin
               IPAddr.new(clean_ip)