datadog 2.23.0 → 2.24.0

data/lib/datadog/core/remote/transport/http/config.rb

@@ -2,11 +2,10 @@
 
 require 'json'
 
-require_relative '../../../transport/http/client'
+require_relative '../../../transport/http/api/endpoint'
+require_relative '../../../transport/http/response'
 require_relative '../../../utils/base64'
 require_relative '../../../utils/truncation'
-require_relative '../../../transport/http/response'
-require_relative '../../../transport/http/api/endpoint'
 
 module Datadog
   module Core
@@ -17,7 +16,7 @@ module Datadog
           module Config
             # Response from HTTP transport for remote configuration
             class Response
-              include Datadog::Core::Transport::HTTP::Response
+              include Core::Transport::HTTP::Response
 
               def initialize(http_response, options = {}) # standard:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
                 super(http_response)
@@ -101,7 +100,7 @@ module Datadog
 
                   {
                     path: h[:path].freeze,
-                    content: StringIO.new(content.freeze),
+                    content: content.freeze,
                   }
                 end.freeze
 
@@ -178,46 +177,7 @@ module Datadog
               end
             end
 
-            # Remote transport HTTP client
-            class Client < Core::Transport::HTTP::Client
-              def send_config_payload(request)
-                send_request(request) do |api, env|
-                  api.send_config(env)
-                end
-              end
-            end
-
             module API
-              # Extensions for HTTP API Spec
-              module Spec
-                attr_reader :config
-
-                def config=(endpoint)
-                  @config = endpoint
-                end
-
-                def send_config(env, &block)
-                  raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('config', self) if config.nil?
-
-                  config.call(env, &block)
-                end
-              end
-
-              # Extensions for HTTP API Instance
-              module Instance
-                def send_config(env)
-                  unless spec.is_a?(Config::API::Spec)
-                    raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new(
-                      'config', self
-                    )
-                  end
-
-                  spec.send_config(env) do |request_env|
-                    call(request_env)
-                  end
-                end
-              end
-
               # Endpoint for remote configuration
               class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
                 HEADER_CONTENT_TYPE = 'Content-Type'

data/lib/datadog/core/remote/transport/http/negotiation.rb

@@ -43,46 +43,7 @@ module Datadog
               attr_reader :version, :endpoints, :config, :span_events
             end
 
-            # Remote negotiation HTTP client
-            class Client < Core::Transport::HTTP::Client
-              def send_info_payload(request)
-                send_request(request) do |api, env|
-                  api.send_info(env)
-                end
-              end
-            end
-
             module API
-              # Extensions for HTTP API Spec
-              module Spec
-                attr_reader :info
-
-                def info=(endpoint)
-                  @info = endpoint
-                end
-
-                def send_info(env, &block)
-                  raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('info', self) if info.nil?
-
-                  info.call(env, &block)
-                end
-              end
-
-              # Extensions for HTTP API Instance
-              module Instance
-                def send_info(env)
-                  unless spec.is_a?(Negotiation::API::Spec)
-                    raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new(
-                      'info', self
-                    )
-                  end
-
-                  spec.send_info(env) do |request_env|
-                    call(request_env)
-                  end
-                end
-              end
-
               # Endpoint for negotiation
               class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
                 def initialize(path)

data/lib/datadog/core/remote/transport/http.rb

@@ -7,16 +7,6 @@ require_relative '../../transport/http'
 require_relative 'config'
 require_relative 'negotiation'
 
-# TODO: Improve negotiation to allow per endpoint selection
-#
-# Since endpoint negotiation happens at the `API::Spec` level there can not be
-# a mix of endpoints at various versions or versionless without describing all
-# the possible combinations as specs. See http/api.
-#
-# Below should be:
-# require_relative '../../transport/http/api'
-require_relative 'http/api'
-
 # TODO: Decouple transport/http
 #
 # Because a new transport is required for every (API, Client, Transport)
@@ -29,26 +19,30 @@ module Datadog
       module Transport
         # Namespace for HTTP transport components
         module HTTP
+          ROOT = Negotiation::API::Endpoint.new(
+            '/info',
+          )
+
+          V7 = Config::API::Endpoint.new(
+            '/v0.7/config',
+            Core::Encoding::JSONEncoder,
+          )
+
           module_function
 
           # Builds a new Transport::HTTP::Client with default settings
           # Pass a block to override any settings.
           def root(
             agent_settings:,
-            logger: Datadog.logger,
-            api_version: nil,
+            logger:,
             headers: nil
           )
             Core::Transport::HTTP.build(
-              api_instance_class: API::Instance,
               agent_settings: agent_settings,
               logger: logger,
-              api_version: api_version,
               headers: headers
             ) do |transport|
-              apis = API.defaults
-
-              transport.api API::ROOT, apis[API::ROOT]
+              transport.api 'root', ROOT
 
               # Call block to apply any customization, if provided
               yield(transport) if block_given?
@@ -59,20 +53,15 @@ module Datadog
           # Pass a block to override any settings.
           def v7(
             agent_settings:,
-            logger: Datadog.logger,
-            api_version: nil,
+            logger:,
             headers: nil
           )
             Core::Transport::HTTP.build(
-              api_instance_class: API::Instance,
               agent_settings: agent_settings,
               logger: logger,
-              api_version: api_version,
               headers: headers
             ) do |transport|
-              apis = API.defaults
-
-              transport.api API::V7, apis[API::V7]
+              transport.api 'v7', V7
 
               # Call block to apply any customization, if provided
               yield(transport) if block_given?

data/lib/datadog/core/remote/transport/negotiation.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../../core/transport/request'
+require_relative '../../../core/transport/transport'
 require_relative 'http/negotiation'
 
 # TODO: Resolve conceptual conundrum
@@ -32,24 +33,14 @@ module Datadog
           end
 
           # Negotiation transport
-          class Transport
-            attr_reader :client, :apis, :default_api, :current_api_id, :logger
-
-            def initialize(apis, default_api, logger: Datadog.logger)
-              @apis = apis
-              @logger = logger
-
-              @client = Remote::Transport::HTTP::Negotiation::Client.new(current_api, logger: logger)
-            end
-
-            def send_info
+          class Transport < Core::Transport::Transport
+            # TODO steep is complaining about this method, I tried to make
+            # it work but between module inclusions and steep diagnostics
+            # I don't understand what the problem is or what the steep wants.
+            def send_info # steep:ignore
               request = Request.new
 
-              @client.send_info_payload(request)
-            end
-
-            def current_api
-              @apis[HTTP::API::ROOT]
+              @client.send_request(:info, request)
             end
           end
         end

data/lib/datadog/core/telemetry/component.rb

@@ -14,14 +14,26 @@ require_relative '../utils/forking'
 module Datadog
   module Core
     module Telemetry
-      # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
-      # Note: Telemetry does not spawn its worker thread in fork processes, thus no telemetry is sent in forked processes.
+      # Telemetry entry point, coordinates sending telemetry events at
+      # various points in application lifecycle.
       #
       # @api private
       class Component
         ENDPOINT_COLLECTION_MESSAGE_LIMIT = 300
 
-        attr_reader :enabled, :logger, :transport, :worker
+        ONLY_ONCE = Utils::OnlyOnce.new
+
+        attr_reader :enabled
+        attr_reader :logger
+        attr_reader :transport
+        attr_reader :worker
+        attr_reader :settings
+        attr_reader :agent_settings
+        attr_reader :metrics_manager
+
+        # Alias for consistency with other components.
+        # TODO Remove +enabled+ method
+        alias_method :enabled?, :enabled
 
         include Core::Utils::Forking
         include Telemetry::Logging
@@ -50,6 +62,17 @@ module Datadog
           logger:,
           enabled:
         )
+          ONLY_ONCE.run do
+            Utils::AtForkMonkeyPatch.apply!
+
+            # All of the other at fork monkey patch callbacks reference
+            # globals, follow that pattern here to avoid having the component
+            # referenced via the at fork callbacks.
+            Datadog::Core::Utils::AtForkMonkeyPatch.at_fork(:child) do
+              Datadog.send(:components, allow_initialization: false)&.telemetry&.after_fork
+            end
+          end
+
           @enabled = enabled
           @log_collection_enabled = settings.telemetry.log_collection_enabled
           @logger = logger
@@ -110,7 +133,7 @@ module Datadog
         end
 
         def start(initial_event_is_change = false, components:)
-          return if !@enabled
+          return unless enabled?
 
           initial_event = if initial_event_is_change
             Event::SynthAppClientConfigurationChange.new(
@@ -136,44 +159,44 @@ module Datadog
         end
 
         def emit_closing!
-          return if !@enabled || forked?
+          return unless enabled?
 
           @worker.enqueue(Event::AppClosing.new)
         end
 
         def integrations_change!
-          return if !@enabled || forked?
+          return unless enabled?
 
           @worker.enqueue(Event::AppIntegrationsChange.new)
         end
 
         def log!(event)
-          return if !@enabled || forked? || !@log_collection_enabled
+          return unless enabled? && @log_collection_enabled
 
           @worker.enqueue(event)
         end
 
         # Wait for the worker to send out all events that have already
         # been queued, up to 15 seconds. Returns whether all events have
-        # been flushed.
+        # been flushed, or nil if telemetry is disabled.
         #
         # @api private
-        def flush
-          return if !@enabled || forked?
+        def flush(timeout: nil)
+          return unless enabled?
 
-          @worker.flush
+          @worker.flush(timeout: timeout)
         end
 
         # Report configuration changes caused by Remote Configuration.
         def client_configuration_change!(changes)
-          return if !@enabled || forked?
+          return unless enabled?
 
           @worker.enqueue(Event::AppClientConfigurationChange.new(changes, 'remote_config'))
         end
 
         # Report application endpoints
         def app_endpoints_loaded(endpoints, page_size: ENDPOINT_COLLECTION_MESSAGE_LIMIT)
-          return if !@enabled || forked?
+          return unless enabled?
 
           endpoints.each_slice(page_size).with_index do |endpoints_slice, i|
             @worker.enqueue(Event::AppEndpointsLoaded.new(endpoints_slice, is_first: i.zero?))
@@ -204,6 +227,22 @@ module Datadog
         def distribution(namespace, metric_name, value, tags: {}, common: true)
           @metrics_manager.distribution(namespace, metric_name, value, tags: tags, common: common)
         end
+
+        # When a fork happens, we generally need to do two things inside the
+        # child proess:
+        # 1. Restart the worker.
+        # 2. Discard any events and metrics that were submitted in the
+        #    parent process (because they will be sent out in the parent
+        #    process, sending them in the child would cause duplicate
+        #    submission).
+        def after_fork
+          # We cannot simply create a new instance of metrics manager because
+          # it is referenced from other objects (e.g. the worker).
+          # We must reset the existing instance.
+          @metrics_manager.clear
+
+          worker&.send(:after_fork_monkey_patched)
+        end
       end
     end
   end

data/lib/datadog/core/telemetry/event/app_started.rb

@@ -11,6 +11,12 @@ module Datadog
           def initialize(components:)
             # To not hold a reference to the component tree, generate
             # the event payload here in the constructor.
+            #
+            # Important: do not store data that contains (or is derived from)
+            # the runtime_id or sequence numbers.
+            # This event is reused when a process forks, but in the
+            # child process the runtime_id would be different and sequence
+            # number is reset.
             @configuration = configuration(components.settings, components.agent_settings)
             @install_signature = install_signature(components.settings)
             @products = products(components)
@@ -30,6 +36,15 @@ module Datadog
             }
           end
 
+          # Whether the event is actually the app-started event.
+          # For the app-started event we follow up by sending
+          # app-dependencies-loaded, if the event is
+          # app-client-configuration-change we don't send
+          # app-dependencies-loaded.
+          def app_started?
+            true
+          end
+
           private
 
           def products(components)
@@ -159,6 +174,25 @@ module Datadog
               get_telemetry_origin(settings, 'tracing.contrib.peer_service_mapping')
             )
 
+            # OpenTelemetry configuration options (using environment variable names)
+            otel_exporter_headers_string = settings.opentelemetry.exporter.headers&.map { |key, value| "#{key}=#{value}" }&.join(',')
+            otel_exporter_metrics_headers_string = settings.opentelemetry.metrics.headers&.map { |key, value| "#{key}=#{value}" }&.join(',')
+            list.push(
+              conf_value('OTEL_EXPORTER_OTLP_ENDPOINT', settings.opentelemetry.exporter.endpoint, seq_id, get_telemetry_origin(settings, 'opentelemetry.exporter.endpoint')),
+              conf_value('OTEL_EXPORTER_OTLP_HEADERS', otel_exporter_headers_string, seq_id, get_telemetry_origin(settings, 'opentelemetry.exporter.headers')),
+              conf_value('OTEL_EXPORTER_OTLP_PROTOCOL', settings.opentelemetry.exporter.protocol, seq_id, get_telemetry_origin(settings, 'opentelemetry.exporter.protocol')),
+              conf_value('OTEL_EXPORTER_OTLP_TIMEOUT', settings.opentelemetry.exporter.timeout_millis, seq_id, get_telemetry_origin(settings, 'opentelemetry.exporter.timeout_millis')),
+              conf_value('DD_METRICS_OTEL_ENABLED', settings.opentelemetry.metrics.enabled, seq_id, get_telemetry_origin(settings, 'opentelemetry.metrics.enabled')),
+              conf_value('OTEL_METRICS_EXPORTER', settings.opentelemetry.metrics.exporter, seq_id, get_telemetry_origin(settings, 'opentelemetry.metrics.exporter')),
+              conf_value('OTEL_EXPORTER_OTLP_METRICS_ENDPOINT', settings.opentelemetry.metrics.endpoint, seq_id, get_telemetry_origin(settings, 'opentelemetry.metrics.endpoint')),
+              conf_value('OTEL_EXPORTER_OTLP_METRICS_HEADERS', otel_exporter_metrics_headers_string, seq_id, get_telemetry_origin(settings, 'opentelemetry.metrics.headers')),
+              conf_value('OTEL_EXPORTER_OTLP_METRICS_PROTOCOL', settings.opentelemetry.metrics.protocol, seq_id, get_telemetry_origin(settings, 'opentelemetry.metrics.protocol')),
+              conf_value('OTEL_EXPORTER_OTLP_METRICS_TIMEOUT', settings.opentelemetry.metrics.timeout_millis, seq_id, get_telemetry_origin(settings, 'opentelemetry.metrics.timeout_millis')),
+              conf_value('OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE', settings.opentelemetry.metrics.temporality_preference, seq_id, get_telemetry_origin(settings, 'opentelemetry.metrics.temporality_preference')),
+              conf_value('OTEL_METRIC_EXPORT_INTERVAL', settings.opentelemetry.metrics.export_interval_millis, seq_id, get_telemetry_origin(settings, 'opentelemetry.metrics.export_interval_millis')),
+              conf_value('OTEL_METRIC_EXPORT_TIMEOUT', settings.opentelemetry.metrics.export_timeout_millis, seq_id, get_telemetry_origin(settings, 'opentelemetry.metrics.export_timeout_millis')),
+            )
+
             # Whitelist of configuration options to send in additional payload object
             TARGET_OPTIONS.each do |option_path|
               split_option = option_path.split('.')

data/lib/datadog/core/telemetry/event/synth_app_client_configuration_change.rb

@@ -28,13 +28,36 @@ module Datadog
         # and app-closing events.
         class SynthAppClientConfigurationChange < AppStarted
           def type
-            'app-client-configuration-change'
+            if reset?
+              super
+            else
+              'app-client-configuration-change'
+            end
           end
 
           def payload
-            {
-              configuration: @configuration,
-            }
+            if reset?
+              super
+            else
+              {
+                configuration: @configuration,
+              }
+            end
+          end
+
+          def app_started?
+            reset?
+          end
+
+          # Revert this event to a "regular" AppStarted event.
+          #
+          # Used in after_fork to send the AppStarted event in child processes.
+          def reset!
+            @reset = true
+          end
+
+          def reset?
+            defined?(@reset) && !!@reset
           end
         end
       end

data/lib/datadog/core/telemetry/metrics_manager.rb

@@ -6,8 +6,11 @@ module Datadog
   module Core
     module Telemetry
       # MetricsManager aggregates and flushes metrics and distributions
+      #
+      # @api private
       class MetricsManager
         attr_reader :enabled
+        attr_reader :collections
 
         def initialize(aggregation_interval:, enabled:)
           @interval = aggregation_interval
@@ -68,6 +71,12 @@ module Datadog
           @enabled = false
         end
 
+        def clear
+          @mutex.synchronize do
+            @collections = {}
+          end
+        end
+
         private
 
         def fetch_or_create_collection(namespace)

data/lib/datadog/core/telemetry/request.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../environment/platform'
+require_relative '../environment/process'
 require_relative '../utils/hash'
 
 module Datadog
@@ -11,9 +12,9 @@ module Datadog
         class << self
           using Core::Utils::Hash::Refinement
 
-          def build_payload(event, seq_id, api_version: 'v2', debug: false)
+          def build_payload(event, seq_id, debug: false)
             hash = {
-              api_version: api_version,
+              api_version: 'v2',
               application: application,
               debug: debug,
               host: host,
@@ -43,7 +44,7 @@ module Datadog
               tracer_version = "#{tracer_version}-ci-#{::Datadog::CI::VERSION::STRING}"
             end
 
-            {
+            app = {
               env: config.env,
               language_name: Core::Environment::Ext::LANG,
               language_version: Core::Environment::Ext::LANG_VERSION,
@@ -53,6 +54,10 @@ module Datadog
               service_version: config.version,
               tracer_version: tracer_version
             }
+
+            tag_process_tags!(app, config)
+
+            app
           end
 
           def host
@@ -64,6 +69,15 @@ module Datadog
               kernel_version: Core::Environment::Platform.kernel_version
             }
           end
+
+          def tag_process_tags!(app, config)
+            return unless config.experimental_propagate_process_tags_enabled
+
+            process_tags = Core::Environment::Process.serialized
+            return if process_tags.empty?
+
+            app[:process_tags] = process_tags
+          end
         end
       end
     end

data/lib/datadog/core/telemetry/transport/http/telemetry.rb

@@ -2,7 +2,6 @@
 
 require_relative '../../../transport/http/api/endpoint'
 require_relative '../../../transport/http/api/instance'
-require_relative '../../../transport/http/api/spec'
 require_relative '../../../transport/http/client'
 require_relative '../../../transport/request'
 
@@ -12,36 +11,7 @@ module Datadog
       module Transport
         module HTTP
           module Telemetry
-            class Client < Core::Transport::HTTP::Client
-              def send_telemetry_payload(request)
-                send_request(request) do |api, env|
-                  # TODO how to make api have the derived type for steep?
-                  api.send_telemetry(env) # steep:ignore
-                end
-              end
-            end
-
             module API
-              class Instance < Core::Transport::HTTP::API::Instance
-                def send_telemetry(env)
-                  raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new('telemetry', self) unless spec.is_a?(Telemetry::API::Spec)
-
-                  spec.send_telemetry(env) do |request_env|
-                    call(request_env)
-                  end
-                end
-              end
-
-              class Spec < Core::Transport::HTTP::API::Spec
-                attr_accessor :telemetry
-
-                def send_telemetry(env, &block)
-                  raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('telemetry', self) if telemetry.nil?
-
-                  telemetry.call(env, &block)
-                end
-              end
-
               class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
                 HEADER_CONTENT_TYPE = 'Content-Type'
 
@@ -65,12 +35,12 @@ module Datadog
                   super
                 end
 
-                def headers(request_type:, api_key:, api_version: 'v2')
+                def headers(request_type:, api_key:)
                   {
                     Core::Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST => '1',
                     # Provided by encoder
                     # 'Content-Type' => 'application/json',
-                    'DD-Telemetry-API-Version' => api_version,
+                    'DD-Telemetry-API-Version' => 'v2',
                     'DD-Telemetry-Request-Type' => request_type,
                     'DD-Client-Library-Language' => Core::Environment::Ext::LANG,
                     'DD-Client-Library-Version' => Core::Environment::Identity.gem_datadog_version_semver2,