datadog 2.23.0 → 2.24.0

data/lib/datadog/core/telemetry/transport/http.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require_relative 'telemetry'
-require_relative 'http/api'
+require_relative '../../encoding'
 require_relative '../../transport/http'
+require_relative 'telemetry'
 
 module Datadog
   module Core
@@ -10,6 +10,16 @@ module Datadog
       module Transport
         # Namespace for HTTP transport components
         module HTTP
+          AGENT_TELEMETRY = Telemetry::API::Endpoint.new(
+            '/telemetry/proxy/api/v2/apmtelemetry',
+            Core::Encoding::JSONEncoder,
+          )
+
+          AGENTLESS_TELEMETRY = Telemetry::API::Endpoint.new(
+            '/api/v2/apmtelemetry',
+            Core::Encoding::JSONEncoder,
+          )
+
           module_function
 
           # Builds a new Transport::HTTP::Client with default settings
@@ -18,18 +28,14 @@ module Datadog
             agent_settings:,
             logger:,
             api_key: nil,
-            api_version: nil,
             headers: nil
           )
-            Core::Transport::HTTP.build(api_instance_class: Telemetry::API::Instance,
+            Core::Transport::HTTP.build(
               logger: logger,
               agent_settings: agent_settings,
-              api_version: api_version,
-              headers: headers) do |transport|
-              apis = API.defaults
-
-              transport.api API::AGENTLESS_TELEMETRY, apis[API::AGENTLESS_TELEMETRY]
-
+              headers: headers
+            ) do |transport|
+              transport.api 'agentless_telemetry', AGENTLESS_TELEMETRY
               # Call block to apply any customization, if provided
               yield(transport) if block_given?
             end.to_transport(Core::Telemetry::Transport::Telemetry::Transport).tap do |transport|
@@ -42,15 +48,14 @@ module Datadog
           def agent_telemetry(
             agent_settings:,
             logger:,
-            api_version: nil,
             headers: nil
           )
-            Core::Transport::HTTP.build(api_instance_class: Telemetry::API::Instance,
+            Core::Transport::HTTP.build(
               logger: logger,
-              agent_settings: agent_settings, api_version: api_version, headers: headers) do |transport|
-              apis = API.defaults
-
-              transport.api API::AGENT_TELEMETRY, apis[API::AGENT_TELEMETRY]
+              agent_settings: agent_settings,
+              headers: headers
+            ) do |transport|
+              transport.api 'agent_telemetry', AGENT_TELEMETRY
 
               # Call block to apply any customization, if provided
               yield(transport) if block_given?

data/lib/datadog/core/telemetry/transport/telemetry.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../transport/parcel'
+require_relative '../../transport/transport'
 require_relative 'http/telemetry'
 
 module Datadog
@@ -23,23 +24,15 @@ module Datadog
             end
           end
 
-          class Transport
-            attr_reader :client, :apis, :default_api, :current_api_id, :logger
+          class Transport < Core::Transport::Transport
             attr_accessor :api_key
 
-            def initialize(apis, default_api, logger:)
-              @apis = apis
-              @logger = logger
-
-              @client = Core::Telemetry::Transport::HTTP::Telemetry::Client.new(@apis[default_api], logger: logger)
-            end
-
             def send_telemetry(request_type:, payload:)
               json = JSON.dump(payload)
               parcel = EncodedParcel.new(json)
               request = Request.new(request_type, parcel, api_key)
 
-              @client.send_telemetry_payload(request)
+              @client.send_request(:telemetry, request)
               # Perform no error checking here
             end
           end

data/lib/datadog/core/telemetry/worker.rb

@@ -9,7 +9,10 @@ require_relative '../workers/queue'
 module Datadog
   module Core
     module Telemetry
-      # Accumulates events and sends them to the API at a regular interval, including heartbeat event.
+      # Accumulates events and sends them to the API at a regular interval,
+      # including heartbeat event.
+      #
+      # @api private
       class Worker
         include Core::Workers::Queue
         include Core::Workers::Polling
@@ -40,11 +43,23 @@ module Datadog
           self.enabled = enabled
           # Workers::IntervalLoop settings
           self.loop_base_interval = metrics_aggregation_interval_seconds
-          self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_STOP
+          # We actually restart the worker after fork, but this is done
+          # via the AtForkMonkeyPatch rather than the worker fork policy
+          # because we also need to reset state outside of the worker
+          # (e.g. the metrics).
+          self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_RESTART
 
           @shutdown_timeout = shutdown_timeout
           @buffer_size = buffer_size
 
+          initialize_state
+        end
+
+        # To make the method calls clear, the initialization code is in this
+        # method called +initialize_state+ which is called from +after_fork+.
+        # This way users of this class (e.g. telemetry Component) do not
+        # need to invoke +initialize_state+ directly, which can be confusing.
+        private def initialize_state
           self.buffer = buffer_klass.new(@buffer_size)
 
           @initial_event_once = Utils::OnlyOnceSuccessful.new(APP_STARTED_EVENT_RETRIES)
@@ -53,12 +68,13 @@ module Datadog
         attr_reader :logger
         attr_reader :initial_event_once
         attr_reader :initial_event
+        attr_reader :emitter
 
         # Returns true if worker thread is successfully started,
         # false if worker thread was not started but telemetry is enabled,
         # nil if telemetry is disabled.
         def start(initial_event)
-          return if !enabled? || forked?
+          return unless enabled?
 
           @initial_event = initial_event
 
@@ -79,7 +95,7 @@ module Datadog
         # for not enqueueing event (presently) is that telemetry is disabled
         # altogether, and in this case other methods return nil.
         def enqueue(event)
-          return if !enabled? || forked?
+          return unless enabled?
 
           buffer.push(event)
           true
@@ -102,38 +118,16 @@ module Datadog
         # been flushed.
         #
         # @api private
-        def flush
-          return true unless enabled? || !run_loop?
-
-          started = Utils::Time.get_time
-          loop do
-            # The AppStarted event is triggered by the worker itself,
-            # from the worker thread. As such the main thread has no way
-            # to delay itself until that event is queued and we need some
-            # way to wait until that event is sent out to assert on it in
-            # the test suite. Check the run once flag which *should*
-            # indicate the event has been queued (at which point our queue
-            # depth check should waint until it's sent).
-            # This is still a hack because the flag can be overridden
-            # either way with or without the event being sent out.
-            # Note that if the AppStarted sending fails, this check
-            # will return false and flushing will be blocked until the
-            # 15 second timeout.
-            # Note that the first wait interval between telemetry event
-            # sending is 10 seconds, the timeout needs to be strictly
-            # greater than that.
-            return true if buffer.empty? && !in_iteration? && sent_initial_event?
-
-            sleep 0.5
-
-            return false if Utils::Time.get_time - started > 15
-          end
+        def flush(timeout: nil)
+          # Increase default timeout to 15 seconds - see the comment in
+          # +idle?+ for more details.
+          super(timeout: timeout || 15)
         end
 
         private
 
         def perform(*events)
-          return if !enabled? || forked?
+          return unless enabled?
 
           if need_initial_event?
             started!
@@ -189,7 +183,9 @@ module Datadog
               # dependencies and send the new ones.
               # System tests demand only one instance of this event per
               # dependency.
-              send_event(Event::AppDependenciesLoaded.new) if @dependency_collection && initial_event.class.eql?(Telemetry::Event::AppStarted) # standard:disable Style/ClassEqualityComparison:
+              if @dependency_collection && initial_event.app_started?
+                send_event(Event::AppDependenciesLoaded.new)
+              end
 
               true
             else
@@ -212,6 +208,8 @@ module Datadog
           res
         end
 
+        # This method overrides Queue's dequeue method and does LIFO instead
+        # of FIFO that Queue implements. Why?
         def dequeue
           buffer.pop
         end
@@ -240,6 +238,45 @@ module Datadog
           disable!
         end
 
+        # Call this method in a forked child to reset the state of this worker.
+        #
+        # Discard any accumulated events since they will be sent by
+        # the parent.
+        # Discard any accumulated metrics.
+        # Restart the worker thread, if it was running in the parent process.
+        #
+        # This method cannot be called +after_fork+ because workers define
+        # and call +after_fork+ which is supposed to do different things.
+        def after_fork_monkey_patched
+          # If telemetry is disabled, we still reset the state to avoid
+          # having wrong state. It is possible that in the future telemetry
+          # will be re-enabled after errors.
+          initialize_state
+          # In the child process, we get a new runtime_id.
+          # As such we need to send AppStarted event.
+          # In the parent process, the event may have been the
+          # SynthAppClientConfigurationChange instead of AppStarted,
+          # and in that case we need to convert it to the "regular"
+          # AppStarted event.
+          if defined?(@initial_event) && @initial_event.is_a?(Event::SynthAppClientConfigurationChange)
+            # It would be great to just replace the initial event in
+            # +initialize_state+ method. Unfortunately this event requires
+            # the entire component tree to build its payload, which we
+            # 1) don't currently have in telemetry and
+            # 2) don't want to keep a permanent reference to in any case.
+            # Therefore we have this +reset!+ method that changes the
+            # event type while keeping the payload.
+            @initial_event.reset! # steep:ignore NoMethod
+          end
+
+          if enabled? && !worker.nil?
+            # Start the background thread if it was started in the parent
+            # process (which requires telemetry to be enabled).
+            # This should be done after all of the state resets.
+            perform
+          end
+        end
+
         # Deduplicate logs by counting the number of repeated occurrences of the same log
         # entry and replacing them with a single entry with the calculated `count` value.
         # Non-log events are unchanged.
@@ -270,6 +307,25 @@ module Datadog
 
           other_events + uniq_logs
         end
+
+        def idle?
+          # The AppStarted event is triggered by the worker itself,
+          # from the worker thread. As such the main thread has no way
+          # to delay itself until that event is queued and we need some
+          # way to wait until that event is sent out to assert on it in
+          # the test suite. Check the run once flag which *should*
+          # indicate the event has been queued (at which point our queue
+          # depth check should wait until it's sent).
+          # This is still a hack because the flag can be overridden
+          # either way with or without the event being sent out.
+          # Note that if the AppStarted sending fails, this check
+          # will return false and flushing will be blocked until the
+          # 15 second timeout.
+          # Note that the first wait interval between telemetry event
+          # sending is 10 seconds, the timeout needs to be strictly
+          # greater than that.
+          super && sent_initial_event?
+        end
       end
     end
   end

data/lib/datadog/core/transport/ext.rb

@@ -8,6 +8,8 @@ module Datadog
       module Ext
         module HTTP
           HEADER_CONTAINER_ID = 'Datadog-Container-ID'
+          HEADER_ENTITY_ID = 'Datadog-Entity-ID'
+          HEADER_EXTERNAL_ENV = 'Datadog-External-Env'
           HEADER_DD_API_KEY = 'DD-API-KEY'
           # Tells agent that `_dd.top_level` metrics have been set by the tracer.
           # The agent will not calculate top-level spans but instead trust the tracer tagging.

data/lib/datadog/core/transport/http/api/endpoint.rb

@@ -9,13 +9,18 @@ module Datadog
         module API
           # Endpoint
           class Endpoint
-            attr_reader \
-              :verb,
-              :path
+            attr_reader :verb
+            attr_reader :path
 
-            def initialize(verb, path)
+            # TODO Currently only Traces transport specifies an encoder.
+            # Other transports perform encoding "inline" / ad-hoc.
+            # They should probably use this encoder field instead.
+            attr_reader :encoder
+
+            def initialize(verb, path, encoder: nil)
               @verb = verb
               @path = path
+              @encoder = encoder
             end
 
             def call(env)

data/lib/datadog/core/transport/http/api/instance.rb

@@ -7,36 +7,19 @@ module Datadog
         module API
           # An API configured with adapter and routes
           class Instance
-            # Raised when an endpoint is invoked on an API that is not the
-            # of expected API class for that endpoint.
-            class EndpointNotSupportedError < StandardError
-              attr_reader :spec, :endpoint_name
-
-              def initialize(endpoint_name, spec)
-                @spec = spec
-                @endpoint_name = endpoint_name
-
-                super(message)
-              end
-
-              def message
-                "#{endpoint_name} not supported for this API!"
-              end
-            end
-
             attr_reader \
               :adapter,
               :headers,
-              :spec
+              :endpoint
 
-            def initialize(spec, adapter, options = {})
-              @spec = spec
+            def initialize(endpoint, adapter, options = {})
+              @endpoint = endpoint
               @adapter = adapter
               @headers = options.fetch(:headers, {})
             end
 
             def encoder
-              spec.encoder
+              endpoint.encoder
             end
 
             def call(env)

data/lib/datadog/core/transport/http/builder.rb

@@ -13,7 +13,6 @@ module Datadog
           REGISTRY = Datadog::Core::Transport::HTTP::Adapters::Registry.new
 
           attr_reader \
-            :api_instance_class,
             :apis,
             :api_options,
             :default_adapter,
@@ -21,7 +20,7 @@ module Datadog
             :default_headers,
             :logger
 
-          def initialize(api_instance_class:, logger: Datadog.logger)
+          def initialize(logger: Datadog.logger)
             # Global settings
             @default_adapter = nil
             @default_headers = {}
@@ -33,7 +32,6 @@ module Datadog
             # API settings
             @api_options = {}
 
-            @api_instance_class = api_instance_class
             @logger = logger
 
             yield(self) if block_given?
@@ -73,6 +71,12 @@ module Datadog
             @apis[key] = spec
 
             # Apply as default API, if specified to do so.
+            #
+            # This code also sets the first defined API to be the default API.
+            # In APIs without fallbacks (currently, everything other than
+            # tracing's Traces, though DI Input will also have fallbacks soon)
+            # there is only one declaration of `transport.api`, and that
+            # API is set as the default API in the transport by this line.
             @default_api = key if options.delete(:default) || @default_api.nil?
 
             # Save all other settings for initialization
@@ -94,7 +98,7 @@ module Datadog
           def to_api_instances
             raise NoApisError if @apis.empty?
 
-            @apis.inject(Datadog::Core::Transport::HTTP::API::Map.new) do |instances, (key, spec)|
+            @apis.inject(API::Map.new) do |instances, (key, spec)|
               instances.tap do
                 api_options = @api_options[key].dup
 
@@ -107,7 +111,7 @@ module Datadog
                 api_options[:headers] = @default_headers.merge((api_options[:headers] || {}))
 
                 # Add API::Instance with all settings
-                instances[key] = api_instance_class.new(
+                instances[key] = API::Instance.new(
                   spec,
                   adapter,
                   api_options

data/lib/datadog/core/transport/http/client.rb

@@ -11,21 +11,32 @@ module Datadog
         #
         # @api private
         class Client
-          attr_reader :api, :logger
+          attr_reader :instance, :logger
 
-          def initialize(api, logger:)
-            @api = api
+          def initialize(instance, logger:)
+            @instance = instance
             @logger = logger
           end
 
-          private
-
-          def send_request(request, &block)
+          def send_request(action, request)
             # Build request into env
             env = build_env(request)
 
-            # Get responses from API
-            yield(api, env).tap do |response|
+            # Get responses from API.
+            # All of our APIs send only one type of request each.
+            instance.endpoint.call(env) do |request_env|
+              instance.call(request_env)
+            end.tap do |response|
+              unless response.ok?
+                # This logging is on debug level.
+                # To report the failed operations on lower levels,
+                # throttling needs to be implemented because
+                # agent unavailability can produce a lot of spam that would
+                # be not desired by customers.
+                # Some transports do report failed operations on warn level
+                # with such throttling.
+                logger.debug { "send_request #{action.inspect} failed: #{response.inspect}" }
+              end
               on_response(response)
             end
           rescue => exception

data/lib/datadog/core/transport/http.rb

@@ -12,16 +12,16 @@ module Datadog
       module HTTP
         # Add adapters to registry
         Builder::REGISTRY.set(
-          Transport::HTTP::Adapters::Net,
+          Core::Transport::HTTP::Adapters::Net,
           Core::Configuration::Ext::Agent::HTTP::ADAPTER
         )
         Builder::REGISTRY.set(
-          Transport::HTTP::Adapters::Test,
-          Transport::Ext::Test::ADAPTER
+          Core::Transport::HTTP::Adapters::Test,
+          Core::Transport::Ext::Test::ADAPTER
         )
         Builder::REGISTRY.set(
-          Transport::HTTP::Adapters::UnixSocket,
-          Transport::Ext::UnixSocket::ADAPTER
+          Core::Transport::HTTP::Adapters::UnixSocket,
+          Core::Transport::Ext::UnixSocket::ADAPTER
         )
 
         module_function
@@ -29,8 +29,13 @@ module Datadog
         # Helper function that delegates to Builder.new
         # but is under HTTP namespace so that client code requires this file
         # to get the adapters configured, and not the builder directly.
-        def build(api_instance_class:, agent_settings:, logger: Datadog.logger, api_version: nil, headers: nil, &block)
-          Builder.new(api_instance_class: api_instance_class, logger: logger) do |transport|
+        def build(
+          agent_settings:,
+          logger: Datadog.logger,
+          headers: nil,
+          &block
+        )
+          Builder.new(logger: logger) do |transport|
             transport.adapter(agent_settings)
             transport.headers(default_headers)
 
@@ -38,34 +43,32 @@ module Datadog
             yield transport
 
             # Apply any settings given by options
-            transport.default_api = api_version if api_version
             transport.headers(headers) if headers
           end
         end
 
         def default_headers
           {
-            Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_TOP_LEVEL => '1',
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG =>
+            Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_TOP_LEVEL => '1',
+            Core::Transport::Ext::HTTP::HEADER_META_LANG =>
               Datadog::Core::Environment::Ext::LANG,
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_VERSION =>
+            Core::Transport::Ext::HTTP::HEADER_META_LANG_VERSION =>
               Datadog::Core::Environment::Ext::LANG_VERSION,
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER =>
+            Core::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER =>
               Datadog::Core::Environment::Ext::LANG_INTERPRETER,
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER_VENDOR =>
+            Core::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER_VENDOR =>
               Core::Environment::Ext::LANG_ENGINE,
-            Datadog::Core::Transport::Ext::HTTP::HEADER_META_TRACER_VERSION =>
+            Core::Transport::Ext::HTTP::HEADER_META_TRACER_VERSION =>
               Datadog::Core::Environment::Ext::GEM_DATADOG_VERSION
           }.tap do |headers|
-            # Add container ID, if present.
-            if (container_id = Datadog::Core::Environment::Container.container_id)
-              headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CONTAINER_ID] = container_id
-            end
+            # Add application container info
+            headers.merge!(Core::Environment::Container.to_headers)
+
             # TODO: inject configuration rather than reading from global here
             unless Datadog.configuration.apm.tracing.enabled
               # Sending this header to the agent will disable metrics computation (and billing) on the agent side
               # by pretending it has already been done on the library side.
-              headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_STATS] = 'yes'
+              headers[Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_STATS] = 'yes'
             end
           end
         end

data/lib/datadog/core/transport/response.rb

@@ -37,6 +37,15 @@ module Datadog
           maybe_code = if respond_to?(:code)
             " code:#{code}," # steep:ignore
           end
+          payload = self.payload
+          # Truncation thresholds are arbitrary but we need to truncate the
+          # payload here because outputting multi-MB request body to the
+          # log is not useful.
+          #
+          # Note that payload can be nil here.
+          if payload && payload.length > 2000 # steep:ignore
+            payload = Utils::Truncation.truncate_in_middle(payload, 1500, 500) # steep:ignore
+          end
           "#{self.class} ok?:#{ok?},#{maybe_code} unsupported?:#{unsupported?}, " \
             "not_found?:#{not_found?}, client_error?:#{client_error?}, " \
             "server_error?:#{server_error?}, internal_error?:#{internal_error?}, " \

data/lib/datadog/core/transport/transport.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require_relative 'http/client'
+
+module Datadog
+  module Core
+    module Transport
+      # Raised when configured with an unknown API version
+      class UnknownApiVersionError < StandardError
+        attr_reader :version
+
+        def initialize(version)
+          super
+
+          @version = version
+        end
+
+        def message
+          "No matching transport API for version #{version}!"
+        end
+      end
+
+      # Raised when the API verson to downgrade to does not map to a
+      # defined API.
+      class NoDowngradeAvailableError < StandardError
+        attr_reader :version
+
+        def initialize(version)
+          super
+
+          @version = version
+        end
+
+        def message
+          "No downgrade from transport API version #{version} is available!"
+        end
+      end
+
+      # Base class for transports.
+      class Transport
+        attr_reader :client, :apis, :default_api, :current_api_id, :logger
+
+        class << self
+          # The HTTP client class to use for requests, derived from
+          # Core::Transport::HTTP::Client.
+          #
+          # Important: this attribute is NOT inherited by derived classes -
+          # it must be set by every Transport class that wants to have a
+          # non-default HTTP::Client instance.
+          attr_accessor :http_client_class
+        end
+
+        def initialize(apis, default_api, logger:)
+          @apis = apis
+          @default_api = default_api
+          @logger = logger
+
+          set_api!(default_api)
+        end
+
+        def current_api
+          apis[current_api_id]
+        end
+
+        private
+
+        def set_api!(api_id)
+          raise UnknownApiVersionError, api_id unless apis.key?(api_id)
+
+          @current_api_id = api_id
+          client_class = self.class.http_client_class || Core::Transport::HTTP::Client
+          @client = client_class.new(current_api, logger: logger) # steep:ignore
+        end
+
+        def downgrade?(response)
+          return false unless apis.fallbacks.key?(current_api_id)
+
+          response.not_found? || response.unsupported?
+        end
+
+        def downgrade!
+          downgrade_api_id = apis.fallbacks[current_api_id]
+          raise NoDowngradeAvailableError, current_api_id if downgrade_api_id.nil?
+
+          set_api!(downgrade_api_id)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/utils/only_once_successful.rb

@@ -67,7 +67,9 @@ module Datadog
 
         private
 
+        # Use this method only after checking that limit is not nil.
         def check_limit!
+          # @type ivar @limit: Integer
           if @retries >= @limit
             @failed = true
             @ran_once = true