datadog 2.21.0 → 2.38.0

data/lib/datadog/core/telemetry/component.rb

@@ -14,12 +14,24 @@ require_relative '../utils/forking'
 module Datadog
   module Core
     module Telemetry
-      # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
-      # Note: Telemetry does not spawn its worker thread in fork processes, thus no telemetry is sent in forked processes.
+      # Telemetry entry point, coordinates sending telemetry events at
+      # various points in application lifecycle.
       #
       # @api private
       class Component
-        attr_reader :enabled, :logger, :transport, :worker
+        ENDPOINT_COLLECTION_MESSAGE_LIMIT = 300
+
+        attr_reader :enabled
+        attr_reader :logger
+        attr_reader :transport
+        attr_reader :worker
+        attr_reader :settings
+        attr_reader :agent_settings
+        attr_reader :metrics_manager
+
+        # Alias for consistency with other components.
+        # TODO Remove +enabled+ method
+        alias_method :enabled?, :enabled
 
         include Core::Utils::Forking
         include Telemetry::Logging
@@ -87,6 +99,7 @@ module Datadog
           @worker = Telemetry::Worker.new(
             enabled: @enabled,
             heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
+            extended_heartbeat_interval_seconds: settings.telemetry.extended_heartbeat_interval_seconds,
             metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
             emitter: Emitter.new(
               @transport,
@@ -96,6 +109,8 @@ module Datadog
             metrics_manager: @metrics_manager,
             dependency_collection: settings.telemetry.dependency_collection,
             logger: logger,
+            settings: settings,
+            agent_settings: agent_settings,
             shutdown_timeout: settings.telemetry.shutdown_timeout_seconds,
           )
 
@@ -107,13 +122,17 @@ module Datadog
           @worker&.enabled = false
         end
 
-        def start(initial_event_is_change = false)
-          return if !@enabled
+        def start(initial_event_is_change = false, components:)
+          return unless enabled?
 
           initial_event = if initial_event_is_change
-            Event::SynthAppClientConfigurationChange.new(agent_settings: @agent_settings)
+            Event::SynthAppClientConfigurationChange.new(
+              components: components,
+            )
           else
-            Event::AppStarted.new(agent_settings: @agent_settings)
+            Event::AppStarted.new(
+              components: components,
+            )
           end
 
           @worker.start(initial_event)
@@ -130,41 +149,50 @@ module Datadog
         end
 
         def emit_closing!
-          return if !@enabled || forked?
+          return unless enabled?
 
           @worker.enqueue(Event::AppClosing.new)
         end
 
         def integrations_change!
-          return if !@enabled || forked?
+          return unless enabled?
 
           @worker.enqueue(Event::AppIntegrationsChange.new)
         end
 
         def log!(event)
-          return if !@enabled || forked? || !@log_collection_enabled
+          return unless enabled? && @log_collection_enabled
 
           @worker.enqueue(event)
         end
 
         # Wait for the worker to send out all events that have already
         # been queued, up to 15 seconds. Returns whether all events have
-        # been flushed.
+        # been flushed, or nil if telemetry is disabled.
         #
         # @api private
-        def flush
-          return if !@enabled || forked?
+        def flush(timeout: nil)
+          return unless enabled?
 
-          @worker.flush
+          @worker.flush(timeout: timeout)
         end
 
         # Report configuration changes caused by Remote Configuration.
         def client_configuration_change!(changes)
-          return if !@enabled || forked?
+          return unless enabled?
 
           @worker.enqueue(Event::AppClientConfigurationChange.new(changes, 'remote_config'))
         end
 
+        # Report application endpoints
+        def app_endpoints_loaded(endpoints, page_size: ENDPOINT_COLLECTION_MESSAGE_LIMIT)
+          return unless enabled?
+
+          endpoints.each_slice(page_size).with_index do |endpoints_slice, i|
+            @worker.enqueue(Event::AppEndpointsLoaded.new(endpoints_slice, is_first: i.zero?))
+          end
+        end
+
         # Increments a count metric.
         def inc(namespace, metric_name, value, tags: {}, common: true)
           @metrics_manager.inc(namespace, metric_name, value, tags: tags, common: common)
@@ -189,6 +217,22 @@ module Datadog
         def distribution(namespace, metric_name, value, tags: {}, common: true)
           @metrics_manager.distribution(namespace, metric_name, value, tags: tags, common: common)
         end
+
+        # When a fork happens, we generally need to do two things inside the
+        # child proess:
+        # 1. Restart the worker.
+        # 2. Discard any events and metrics that were submitted in the
+        #    parent process (because they will be sent out in the parent
+        #    process, sending them in the child would cause duplicate
+        #    submission).
+        def after_fork
+          # We cannot simply create a new instance of metrics manager because
+          # it is referenced from other objects (e.g. the worker).
+          # We must reset the existing instance.
+          @metrics_manager.clear
+
+          worker&.send(:after_fork_monkey_patched)
+        end
       end
     end
   end

data/lib/datadog/core/telemetry/emitter.rb

@@ -31,15 +31,15 @@ module Datadog
           seq_id = self.class.sequence.next
           payload = Request.build_payload(event, seq_id, debug: debug?)
           res = @transport.send_telemetry(request_type: event.type, payload: payload)
-          logger.debug { "Telemetry sent for event `#{event.type}` (response code: #{res.code})" }
+          if res.ok?
+            logger.debug { "Telemetry sent for event `#{event.type}`" }
+          else
+            logger.debug { "Failed to send telemetry for event `#{event.type}`: #{res.inspect}" }
+          end
           res
         rescue => e
           logger.debug {
-            "Unable to send telemetry request for event `#{begin
-              event.type
-            rescue
-              "unknown"
-            end}`: #{e}"
+            "Unable to send telemetry request for event `#{event.respond_to?(:type) ? event.type : event.to_s}`: #{e.class}: #{e.message}"
           }
           Core::Transport::InternalErrorResponse.new(e)
         end

data/lib/datadog/core/telemetry/event/app_client_configuration_change.rb

@@ -26,14 +26,13 @@ module Datadog
 
           def configuration
             config = Datadog.configuration
-            seq_id = Event.configuration_sequence.next
 
             res = @changes.map do |name, value|
               {
                 name: name,
                 value: value,
                 origin: @origin,
-                seq_id: seq_id,
+                seq_id: Configuration::Option::Precedence::REMOTE_CONFIGURATION.numeric.next,
               }
             end
 
@@ -43,7 +42,7 @@ module Datadog
                 name: 'appsec.sca_enabled',
                 value: config.appsec.sca_enabled,
                 origin: 'code',
-                seq_id: seq_id,
+                seq_id: Configuration::Option::Precedence::PROGRAMMATIC.numeric.next,
               }
             end
 

data/lib/datadog/core/telemetry/event/app_endpoints_loaded.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry event class for sending 'app-endpoints' payload
+        class AppEndpointsLoaded < Base
+          def initialize(endpoints, is_first:)
+            @endpoints = endpoints
+            @is_first = !!is_first
+          end
+
+          def type
+            'app-endpoints'
+          end
+
+          def payload
+            {
+              is_first: @is_first,
+              endpoints: @endpoints
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event/app_extended_heartbeat.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'app_started'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry class for the 'app-extended-heartbeat' event
+        class AppExtendedHeartbeat < AppStarted
+          def initialize(settings:, agent_settings:)
+            @configuration = configuration(settings, agent_settings)
+          end
+
+          def type
+            'app-extended-heartbeat'
+          end
+
+          def payload
+            {
+              configuration: @configuration,
+            }
+          end
+
+          def app_started?
+            false
+          end
+        end
+      end
+    end
+  end
+end