datadog 2.12.0 → 2.22.0

data/lib/datadog/core/remote/transport/http.rb

@@ -1,14 +1,9 @@
 # frozen_string_literal: true
 
-require 'uri'
-
 require_relative '../../environment/container'
 require_relative '../../environment/ext'
 require_relative '../../transport/ext'
-require_relative '../../transport/http/builder'
-require_relative '../../transport/http/adapters/net'
-require_relative '../../transport/http/adapters/unix_socket'
-require_relative '../../transport/http/adapters/test'
+require_relative '../../transport/http'
 
 # TODO: Improve negotiation to allow per endpoint selection
 #
@@ -32,114 +27,55 @@ module Datadog
       module Transport
         # Namespace for HTTP transport components
         module HTTP
-          # NOTE: Due to... legacy reasons... This class likes having a default `AgentSettings` instance to fall back to.
-          # Because we generate this instance with an empty instance of `Settings`, the resulting `AgentSettings` below
-          # represents only settings specified via environment variables + the usual defaults.
-          #
-          # DO NOT USE THIS IN NEW CODE, as it ignores any settings specified by users via `Datadog.configure`.
-          DO_NOT_USE_ENVIRONMENT_AGENT_SETTINGS = Datadog::Core::Configuration::AgentSettingsResolver.call(
-            Datadog::Core::Configuration::Settings.new,
-            logger: nil,
-          )
-
           module_function
 
-          # Builds a new Transport::HTTP::Client
-          def new(klass, &block)
-            Core::Transport::HTTP::Builder.new(
-              api_instance_class: API::Instance, &block
-            ).to_transport(klass)
-          end
-
           # Builds a new Transport::HTTP::Client with default settings
           # Pass a block to override any settings.
           def root(
-            agent_settings: DO_NOT_USE_ENVIRONMENT_AGENT_SETTINGS,
-            **options
+            agent_settings:,
+            logger: Datadog.logger,
+            api_version: nil,
+            headers: nil
           )
-            new(Core::Remote::Transport::Negotiation::Transport) do |transport|
-              transport.adapter(agent_settings)
-              transport.headers(default_headers)
-
+            Core::Transport::HTTP.build(
+              api_instance_class: API::Instance,
+              agent_settings: agent_settings,
+              logger: logger,
+              api_version: api_version,
+              headers: headers
+            ) do |transport|
               apis = API.defaults
 
               transport.api API::ROOT, apis[API::ROOT]
 
-              # Apply any settings given by options
-              unless options.empty?
-                transport.default_api = options[:api_version] if options.key?(:api_version)
-                transport.headers options[:headers] if options.key?(:headers)
-              end
-
               # Call block to apply any customization, if provided
               yield(transport) if block_given?
-            end
+            end.to_transport(Core::Remote::Transport::Negotiation::Transport)
           end
 
           # Builds a new Transport::HTTP::Client with default settings
           # Pass a block to override any settings.
           def v7(
-            agent_settings: DO_NOT_USE_ENVIRONMENT_AGENT_SETTINGS,
-            **options
+            agent_settings:,
+            logger: Datadog.logger,
+            api_version: nil,
+            headers: nil
           )
-            new(Core::Remote::Transport::Config::Transport) do |transport|
-              transport.adapter(agent_settings)
-              transport.headers default_headers
-
+            Core::Transport::HTTP.build(
+              api_instance_class: API::Instance,
+              agent_settings: agent_settings,
+              logger: logger,
+              api_version: api_version,
+              headers: headers
+            ) do |transport|
               apis = API.defaults
 
               transport.api API::V7, apis[API::V7]
 
-              # Apply any settings given by options
-              unless options.empty?
-                transport.default_api = options[:api_version] if options.key?(:api_version)
-                transport.headers options[:headers] if options.key?(:headers)
-              end
-
               # Call block to apply any customization, if provided
               yield(transport) if block_given?
-            end
+            end.to_transport(Core::Remote::Transport::Config::Transport)
           end
-
-          def default_headers
-            {
-              Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_TOP_LEVEL => '1',
-              Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG => Datadog::Core::Environment::Ext::LANG,
-              Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_VERSION =>
-                Datadog::Core::Environment::Ext::LANG_VERSION,
-              Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER =>
-                Datadog::Core::Environment::Ext::LANG_INTERPRETER,
-              Datadog::Core::Transport::Ext::HTTP::HEADER_META_TRACER_VERSION =>
-                Datadog::Core::Environment::Ext::GEM_DATADOG_VERSION
-            }.tap do |headers|
-              # Add container ID, if present.
-              container_id = Datadog::Core::Environment::Container.container_id
-              headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CONTAINER_ID] = container_id unless container_id.nil?
-              # Sending this header to the agent will disable metrics computation (and billing) on the agent side
-              # by pretending it has already been done on the library side.
-              if Datadog.configuration.appsec.standalone.enabled
-                headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_STATS] = 'yes'
-              end
-            end
-          end
-
-          def default_adapter
-            Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
-          end
-
-          # Add adapters to registry
-          Core::Transport::HTTP::Builder::REGISTRY.set(
-            Datadog::Core::Transport::HTTP::Adapters::Net,
-            Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
-          )
-          Core::Transport::HTTP::Builder::REGISTRY.set(
-            Datadog::Core::Transport::HTTP::Adapters::Test,
-            Datadog::Core::Transport::Ext::Test::ADAPTER
-          )
-          Core::Transport::HTTP::Builder::REGISTRY.set(
-            Datadog::Core::Transport::HTTP::Adapters::UnixSocket,
-            Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
-          )
         end
       end
     end

data/lib/datadog/core/remote/transport/negotiation.rb

@@ -49,12 +49,13 @@ module Datadog
 
           # Negotiation transport
           class Transport
-            attr_reader :client, :apis, :default_api, :current_api_id
+            attr_reader :client, :apis, :default_api, :current_api_id, :logger
 
-            def initialize(apis, default_api)
+            def initialize(apis, default_api, logger: Datadog.logger)
               @apis = apis
+              @logger = logger
 
-              @client = HTTP::Client.new(current_api)
+              @client = HTTP::Client.new(current_api, logger: logger)
             end
 
             def send_info

data/lib/datadog/core/runtime/ext.rb

@@ -13,7 +13,6 @@ module Datadog
         # @public_api
         module Metrics
           ENV_ENABLED = 'DD_RUNTIME_METRICS_ENABLED'
-          ENV_OTEL_METRICS_EXPORTER = 'OTEL_METRICS_EXPORTER'
 
           METRIC_CLASS_COUNT = 'runtime.ruby.class_count'
           METRIC_GC_PREFIX = 'runtime.ruby.gc'

data/lib/datadog/core/runtime/metrics.rb

@@ -14,13 +14,16 @@ module Datadog
     module Runtime
       # For generating runtime metrics
       class Metrics < Core::Metrics::Client
-        def initialize(**options)
+        def initialize(telemetry:, **options)
           super
 
           # Initialize service list
           @services = Set.new(options.fetch(:services, []))
           @service_tags = nil
           compile_service_tags!
+
+          # Initialize the collection of runtime-id
+          @runtime_id_enabled = options.fetch(:experimental_runtime_id_enabled, false)
         end
 
         # Associate service with runtime metrics
@@ -93,7 +96,7 @@ module Datadog
 
         def try_flush
           yield
-        rescue StandardError => e
+        rescue => e
           Datadog.logger.warn("Error while sending runtime metric. Cause: #{e.class.name} #{e.message}")
         end
 
@@ -105,6 +108,9 @@ module Datadog
 
             # Add services dynamically because they might change during runtime.
             options[:tags].concat(service_tags) unless service_tags.nil?
+
+            # Add runtime-id dynamically because it might change during runtime.
+            options[:tags].concat(["runtime-id:#{Core::Environment::Identity.id}"]) if @runtime_id_enabled
           end
         end
 
@@ -112,7 +118,8 @@ module Datadog
 
         attr_reader \
           :service_tags,
-          :services
+          :services,
+          :runtime_id_enabled
 
         def compile_service_tags!
           @service_tags = services.to_a.collect do |service|
@@ -140,7 +147,7 @@ module Datadog
           gauge(metric_name, metric_value) if metric_value
         end
 
-        # rubocop:disable Metrics/MethodLength
+        # standard:disable Metrics/MethodLength
         def flush_yjit_stats
           # Only on Ruby >= 3.2
           try_flush do
@@ -188,7 +195,7 @@ module Datadog
             end
           end
         end
-        # rubocop:enable Metrics/MethodLength
+        # standard:enable Metrics/MethodLength
       end
     end
   end

data/lib/datadog/core/tag_builder.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative 'environment/socket'
+require_relative 'environment/identity'
+require_relative 'environment/git'
+
+module Datadog
+  module Core
+    # This module builds a hash of tags.
+    #
+    # When changing or adding the tags, make sure they are kept in sync with
+    # https://docs.google.com/spreadsheets/d/1LOGMf4c4Avbtn36uZ2SWvhIGKRPLM1BoWkUP4JYj7hA/
+    # (Datadog internal link).
+    #
+    # @api private
+    module TagBuilder
+      def self.fixed_environment_tags
+        @fixed_environment_tags ||= {
+          'language' => Environment::Identity.lang,
+          'runtime' => Environment::Identity.lang, # Known to be duplicated from language above
+          'runtime_engine' => Environment::Identity.lang_engine,
+          'runtime_platform' => Environment::Identity.lang_platform,
+          'runtime_version' => Environment::Identity.lang_version,
+          'library_version' => Environment::Identity.gem_datadog_version,
+          'git.repository_url' => Environment::Git.git_repository_url,
+          'git.commit.sha' => Environment::Git.git_commit_sha,
+        }.compact.freeze
+      end
+
+      def self.tags(settings)
+        # Note that user tags get overwritten by our tags, and also
+        # that user tags do not get compacted (nil values are sent as
+        # empty strings).
+        settings.tags.merge(fixed_environment_tags).merge({
+          # Hostname can possibly change during application runtime.
+          'host' => Environment::Socket.hostname,
+          # Runtime ID changes upon a fork.
+          'runtime-id' => Environment::Identity.id,
+          # Process ID changes upon a fork.
+          'process_id' => Process.pid.to_s,
+          # Unified service tagging.
+          'env' => settings.env,
+          'service' => settings.service,
+          'version' => settings.version,
+        }.compact)
+      end
+
+      def self.serialize_tags(tags)
+        # DEV: Should there be some sort of escaping done here?
+        tags.map do |key, value|
+          "#{key}:#{value}"
+        end.join(',')
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/component.rb

@@ -2,12 +2,13 @@
 
 require_relative 'emitter'
 require_relative 'event'
-require_relative 'http/transport'
 require_relative 'metrics_manager'
 require_relative 'worker'
 require_relative 'logging'
+require_relative 'transport/http'
 
 require_relative '../configuration/ext'
+require_relative '../configuration/agentless_settings_resolver'
 require_relative '../utils/forking'
 
 module Datadog
@@ -15,8 +16,12 @@ module Datadog
     module Telemetry
       # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
       # Note: Telemetry does not spawn its worker thread in fork processes, thus no telemetry is sent in forked processes.
+      #
+      # @api private
       class Component
-        attr_reader :enabled, :logger
+        ENDPOINT_COLLECTION_MESSAGE_LIMIT = 300
+
+        attr_reader :enabled, :logger, :transport, :worker
 
         include Core::Utils::Forking
         include Telemetry::Logging
@@ -25,89 +30,104 @@ module Datadog
           enabled = settings.telemetry.enabled
           agentless_enabled = settings.telemetry.agentless_enabled
 
-          if !agentless_enabled && agent_settings.adapter != Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
-            enabled = false
-            logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
-          end
-
           if agentless_enabled && settings.api_key.nil?
             enabled = false
-            logger.debug { 'Telemetry disabled. Agentless telemetry requires an DD_API_KEY variable to be set.' }
+            logger.debug { 'Telemetry disabled. Agentless telemetry requires a DD_API_KEY variable to be set.' }
           end
 
-          transport = if agentless_enabled
-                        Datadog::Core::Telemetry::Http::Transport.build_agentless_transport(
-                          api_key: settings.api_key,
-                          dd_site: settings.site,
-                          url_override: settings.telemetry.agentless_url_override
-                        )
-                      else
-                        Datadog::Core::Telemetry::Http::Transport.build_agent_transport(agent_settings)
-                      end
-
           Telemetry::Component.new(
-            http_transport: transport,
+            settings: settings,
+            agent_settings: agent_settings,
             enabled: enabled,
-            metrics_enabled: enabled && settings.telemetry.metrics_enabled,
-            heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
-            metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
-            dependency_collection: settings.telemetry.dependency_collection,
             logger: logger,
-            shutdown_timeout_seconds: settings.telemetry.shutdown_timeout_seconds,
-            log_collection_enabled: settings.telemetry.log_collection_enabled
           )
         end
 
         # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
-        # @param metrics_enabled [Boolean] Determines whether telemetry metrics should be sent to the API
-        # @param heartbeat_interval_seconds [Float] How frequently heartbeats will be reported, in seconds.
-        # @param metrics_aggregation_interval_seconds [Float] How frequently metrics will be aggregated, in seconds.
-        # @param [Boolean] dependency_collection Whether to send the `app-dependencies-loaded` event
-        def initialize(
-          heartbeat_interval_seconds:,
-          metrics_aggregation_interval_seconds:,
-          dependency_collection:,
+        def initialize( # standard:disable Metrics/MethodLength
+          settings:,
+          agent_settings:,
           logger:,
-          http_transport:,
-          shutdown_timeout_seconds:,
-          enabled: true,
-          metrics_enabled: true,
-          log_collection_enabled: true
+          enabled:
         )
           @enabled = enabled
-          @log_collection_enabled = log_collection_enabled
+          @log_collection_enabled = settings.telemetry.log_collection_enabled
           @logger = logger
 
           @metrics_manager = MetricsManager.new(
-            enabled: enabled && metrics_enabled,
-            aggregation_interval: metrics_aggregation_interval_seconds
+            enabled: @enabled && settings.telemetry.metrics_enabled,
+            aggregation_interval: settings.telemetry.metrics_aggregation_interval_seconds,
           )
 
+          @stopped = false
+
+          return unless @enabled
+
+          @transport = if settings.telemetry.agentless_enabled
+            # We don't touch the `agent_settings` since we still want the telemetry payloads to refer to the original
+            # settings, even though the telemetry itself may be using a different path.
+            telemetry_specific_agent_settings = Core::Configuration::AgentlessSettingsResolver.call(
+              settings,
+              host_prefix: 'instrumentation-telemetry-intake',
+              url_override: settings.telemetry.agentless_url_override,
+              url_override_source: 'c.telemetry.agentless_url_override',
+              logger: logger,
+            )
+            Telemetry::Transport::HTTP.agentless_telemetry(
+              agent_settings: telemetry_specific_agent_settings,
+              logger: logger,
+              # api_key should have already validated to be
+              # not nil by +build+ method above.
+              api_key: settings.api_key,
+            )
+          else
+            Telemetry::Transport::HTTP.agent_telemetry(
+              agent_settings: agent_settings, logger: logger,
+            )
+          end
+
           @worker = Telemetry::Worker.new(
             enabled: @enabled,
-            heartbeat_interval_seconds: heartbeat_interval_seconds,
-            metrics_aggregation_interval_seconds: metrics_aggregation_interval_seconds,
-            emitter: Emitter.new(http_transport: http_transport),
+            heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
+            metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
+            emitter: Emitter.new(
+              @transport,
+              logger: @logger,
+              debug: settings.telemetry.debug,
+            ),
             metrics_manager: @metrics_manager,
-            dependency_collection: dependency_collection,
+            dependency_collection: settings.telemetry.dependency_collection,
             logger: logger,
-            shutdown_timeout: shutdown_timeout_seconds
+            shutdown_timeout: settings.telemetry.shutdown_timeout_seconds,
           )
 
-          @stopped = false
-
-          @worker.start
+          @agent_settings = agent_settings
         end
 
         def disable!
           @enabled = false
-          @worker.enabled = false
+          @worker&.enabled = false
         end
 
-        def stop!
+        def start(initial_event_is_change = false)
+          return if !@enabled
+
+          initial_event = if initial_event_is_change
+            Event::SynthAppClientConfigurationChange.new(agent_settings: @agent_settings)
+          else
+            Event::AppStarted.new(agent_settings: @agent_settings)
+          end
+
+          @worker.start(initial_event)
+        end
+
+        def shutdown!
           return if @stopped
 
-          @worker.stop(true)
+          if defined?(@worker)
+            @worker&.stop(true)
+          end
+
           @stopped = true
         end
 
@@ -129,6 +149,17 @@ module Datadog
           @worker.enqueue(event)
         end
 
+        # Wait for the worker to send out all events that have already
+        # been queued, up to 15 seconds. Returns whether all events have
+        # been flushed.
+        #
+        # @api private
+        def flush
+          return if !@enabled || forked?
+
+          @worker.flush
+        end
+
         # Report configuration changes caused by Remote Configuration.
         def client_configuration_change!(changes)
           return if !@enabled || forked?
@@ -136,6 +167,15 @@ module Datadog
           @worker.enqueue(Event::AppClientConfigurationChange.new(changes, 'remote_config'))
         end
 
+        # Report application endpoints
+        def app_endpoints_loaded(endpoints, page_size: ENDPOINT_COLLECTION_MESSAGE_LIMIT)
+          return if !@enabled || forked?
+
+          endpoints.each_slice(page_size).with_index do |endpoints_slice, i|
+            @worker.enqueue(Event::AppEndpointsLoaded.new(endpoints_slice, is_first: i.zero?))
+          end
+        end
+
         # Increments a count metric.
         def inc(namespace, metric_name, value, tags: {}, common: true)
           @metrics_manager.inc(namespace, metric_name, value, tags: tags, common: common)

data/lib/datadog/core/telemetry/emitter.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'request'
-require_relative 'http/transport'
+require_relative '../transport/response'
 require_relative '../utils/sequence'
 require_relative '../utils/forking'
 
@@ -10,26 +10,38 @@ module Datadog
     module Telemetry
       # Class that emits telemetry events
       class Emitter
-        attr_reader :http_transport
+        attr_reader :transport, :logger
 
         extend Core::Utils::Forking
 
-        # @param http_transport [Datadog::Core::Telemetry::Http::Transport] Transport object that can be used to send
-        #   telemetry requests via the agent
-        def initialize(http_transport:)
-          @http_transport = http_transport
+        # @param transport [Datadog::Core::Telemetry::Transport::Telemetry::Transport]
+        #   Transport object that can be used to send telemetry requests
+        def initialize(transport, logger: Datadog.logger, debug: false)
+          @transport = transport
+          @logger = logger
+          @debug = !!debug
+        end
+
+        def debug?
+          @debug
         end
 
         # Retrieves and emits a TelemetryRequest object based on the request type specified
         def request(event)
           seq_id = self.class.sequence.next
-          payload = Request.build_payload(event, seq_id)
-          res = @http_transport.request(request_type: event.type, payload: payload.to_json)
-          Datadog.logger.debug { "Telemetry sent for event `#{event.type}` (code: #{res.code.inspect})" }
+          payload = Request.build_payload(event, seq_id, debug: debug?)
+          res = @transport.send_telemetry(request_type: event.type, payload: payload)
+          if res.ok?
+            logger.debug { "Telemetry sent for event `#{event.type}`" }
+          else
+            logger.debug { "Failed to send telemetry for event `#{event.type}`: #{res.inspect}" }
+          end
           res
         rescue => e
-          Datadog.logger.debug("Unable to send telemetry request for event `#{event.type rescue 'unknown'}`: #{e}")
-          Telemetry::Http::InternalErrorResponse.new(e)
+          logger.debug {
+            "Unable to send telemetry request for event `#{event.respond_to?(:type) ? event.type : event.to_s}`: #{e.class}: #{e}"
+          }
+          Core::Transport::InternalErrorResponse.new(e)
         end
 
         # Initializes a Sequence object to track seq_id if not already initialized; else returns stored

data/lib/datadog/core/telemetry/event/app_client_configuration_change.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry class for the 'app-client-configuration-change' event
+        class AppClientConfigurationChange < Base
+          attr_reader :changes, :origin
+
+          def type
+            'app-client-configuration-change'
+          end
+
+          def initialize(changes, origin)
+            super()
+            @changes = changes
+            @origin = origin
+          end
+
+          def payload
+            {configuration: configuration}
+          end
+
+          def configuration
+            config = Datadog.configuration
+            seq_id = Event.configuration_sequence.next
+
+            res = @changes.map do |name, value|
+              {
+                name: name,
+                value: value,
+                origin: @origin,
+                seq_id: seq_id,
+              }
+            end
+
+            # DEV: This seems unnecessary (we send the state of sca_enabled for each remote config change)
+            unless config.dig('appsec', 'sca_enabled').nil?
+              res << {
+                name: 'appsec.sca_enabled',
+                value: config.appsec.sca_enabled,
+                origin: 'code',
+                seq_id: seq_id,
+              }
+            end
+
+            res
+          end
+
+          def ==(other)
+            other.is_a?(AppClientConfigurationChange) && other.changes == @changes && other.origin == @origin
+          end
+
+          alias_method :eql?, :==
+
+          def hash
+            [self.class, @changes, @origin].hash
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event/app_closing.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry class for the 'app-closing' event
+        class AppClosing < Base
+          def type
+            'app-closing'
+          end
+        end
+      end
+    end
+  end
+end