datadog 2.7.1 → 2.17.0

data/lib/datadog/core/telemetry/component.rb

@@ -2,20 +2,24 @@
 
 require_relative 'emitter'
 require_relative 'event'
-require_relative 'http/transport'
 require_relative 'metrics_manager'
 require_relative 'worker'
 require_relative 'logging'
+require_relative 'transport/http'
 
 require_relative '../configuration/ext'
+require_relative '../configuration/agentless_settings_resolver'
 require_relative '../utils/forking'
 
 module Datadog
   module Core
     module Telemetry
       # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
+      # Note: Telemetry does not spawn its worker thread in fork processes, thus no telemetry is sent in forked processes.
+      #
+      # @api private
       class Component
-        attr_reader :enabled
+        attr_reader :enabled, :logger, :transport, :worker
 
         include Core::Utils::Forking
         include Telemetry::Logging
@@ -24,80 +28,100 @@ module Datadog
           enabled = settings.telemetry.enabled
           agentless_enabled = settings.telemetry.agentless_enabled
 
-          if !agentless_enabled && agent_settings.adapter != Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
-            enabled = false
-            logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
-          end
-
           if agentless_enabled && settings.api_key.nil?
             enabled = false
-            logger.debug { 'Telemetry disabled. Agentless telemetry requires an DD_API_KEY variable to be set.' }
+            logger.debug { 'Telemetry disabled. Agentless telemetry requires a DD_API_KEY variable to be set.' }
           end
 
-          transport = if agentless_enabled
-                        Datadog::Core::Telemetry::Http::Transport.build_agentless_transport(
-                          api_key: settings.api_key,
-                          dd_site: settings.site,
-                          url_override: settings.telemetry.agentless_url_override
-                        )
-                      else
-                        Datadog::Core::Telemetry::Http::Transport.build_agent_transport(agent_settings)
-                      end
-
           Telemetry::Component.new(
-            http_transport: transport,
+            settings: settings,
+            agent_settings: agent_settings,
             enabled: enabled,
-            metrics_enabled: enabled && settings.telemetry.metrics_enabled,
-            heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
-            metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
-            dependency_collection: settings.telemetry.dependency_collection,
-            shutdown_timeout_seconds: settings.telemetry.shutdown_timeout_seconds,
+            logger: logger,
           )
         end
 
         # @param enabled [Boolean] Determines whether telemetry events should be sent to the API
-        # @param metrics_enabled [Boolean] Determines whether telemetry metrics should be sent to the API
-        # @param heartbeat_interval_seconds [Float] How frequently heartbeats will be reported, in seconds.
-        # @param metrics_aggregation_interval_seconds [Float] How frequently metrics will be aggregated, in seconds.
-        # @param [Boolean] dependency_collection Whether to send the `app-dependencies-loaded` event
-        def initialize(
-          heartbeat_interval_seconds:,
-          metrics_aggregation_interval_seconds:,
-          dependency_collection:,
-          http_transport:,
-          shutdown_timeout_seconds:,
-          enabled: true,
-          metrics_enabled: true
+        def initialize( # standard:disable Metrics/MethodLength
+          settings:,
+          agent_settings:,
+          logger:,
+          enabled:
         )
           @enabled = enabled
-          @stopped = false
+          @log_collection_enabled = settings.telemetry.log_collection_enabled
+          @logger = logger
 
           @metrics_manager = MetricsManager.new(
-            enabled: enabled && metrics_enabled,
-            aggregation_interval: metrics_aggregation_interval_seconds
+            enabled: @enabled && settings.telemetry.metrics_enabled,
+            aggregation_interval: settings.telemetry.metrics_aggregation_interval_seconds,
           )
 
+          @stopped = false
+
+          return unless @enabled
+
+          @transport = if settings.telemetry.agentless_enabled
+            agent_settings = Core::Configuration::AgentlessSettingsResolver.call(
+              settings,
+              host_prefix: 'instrumentation-telemetry-intake',
+              url_override: settings.telemetry.agentless_url_override,
+              url_override_source: 'c.telemetry.agentless_url_override',
+              logger: logger,
+            )
+            Telemetry::Transport::HTTP.agentless_telemetry(
+              agent_settings: agent_settings,
+              logger: logger,
+              # api_key should have already validated to be
+              # not nil by +build+ method above.
+              api_key: settings.api_key,
+            )
+          else
+            Telemetry::Transport::HTTP.agent_telemetry(
+              agent_settings: agent_settings, logger: logger,
+            )
+          end
+
           @worker = Telemetry::Worker.new(
             enabled: @enabled,
-            heartbeat_interval_seconds: heartbeat_interval_seconds,
-            metrics_aggregation_interval_seconds: metrics_aggregation_interval_seconds,
-            emitter: Emitter.new(http_transport: http_transport),
+            heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
+            metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
+            emitter: Emitter.new(
+              @transport,
+              logger: @logger,
+              debug: settings.telemetry.debug,
+            ),
             metrics_manager: @metrics_manager,
-            dependency_collection: dependency_collection,
-            shutdown_timeout: shutdown_timeout_seconds
+            dependency_collection: settings.telemetry.dependency_collection,
+            logger: logger,
+            shutdown_timeout: settings.telemetry.shutdown_timeout_seconds,
           )
-          @worker.start
         end
 
         def disable!
           @enabled = false
-          @worker.enabled = false
+          @worker&.enabled = false
+        end
+
+        def start(initial_event_is_change = false)
+          return if !@enabled
+
+          initial_event = if initial_event_is_change
+            Event::SynthAppClientConfigurationChange.new
+          else
+            Event::AppStarted.new
+          end
+
+          @worker.start(initial_event)
         end
 
-        def stop!
+        def shutdown!
           return if @stopped
 
-          @worker.stop(true)
+          if defined?(@worker)
+            @worker&.stop(true)
+          end
+
           @stopped = true
         end
 
@@ -114,11 +138,22 @@ module Datadog
         end
 
         def log!(event)
-          return unless @enabled || forked?
+          return if !@enabled || forked? || !@log_collection_enabled
 
           @worker.enqueue(event)
         end
 
+        # Wait for the worker to send out all events that have already
+        # been queued, up to 15 seconds. Returns whether all events have
+        # been flushed.
+        #
+        # @api private
+        def flush
+          return if !@enabled || forked?
+
+          @worker.flush
+        end
+
         # Report configuration changes caused by Remote Configuration.
         def client_configuration_change!(changes)
           return if !@enabled || forked?

data/lib/datadog/core/telemetry/emitter.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'request'
-require_relative 'http/transport'
+require_relative '../transport/response'
 require_relative '../utils/sequence'
 require_relative '../utils/forking'
 
@@ -10,26 +10,38 @@ module Datadog
     module Telemetry
       # Class that emits telemetry events
       class Emitter
-        attr_reader :http_transport
+        attr_reader :transport, :logger
 
         extend Core::Utils::Forking
 
-        # @param http_transport [Datadog::Core::Telemetry::Http::Transport] Transport object that can be used to send
-        #   telemetry requests via the agent
-        def initialize(http_transport:)
-          @http_transport = http_transport
+        # @param transport [Datadog::Core::Telemetry::Transport::Telemetry::Transport]
+        #   Transport object that can be used to send telemetry requests
+        def initialize(transport, logger: Datadog.logger, debug: false)
+          @transport = transport
+          @logger = logger
+          @debug = !!debug
+        end
+
+        def debug?
+          @debug
         end
 
         # Retrieves and emits a TelemetryRequest object based on the request type specified
         def request(event)
           seq_id = self.class.sequence.next
-          payload = Request.build_payload(event, seq_id)
-          res = @http_transport.request(request_type: event.type, payload: payload.to_json)
-          Datadog.logger.debug { "Telemetry sent for event `#{event.type}` (code: #{res.code.inspect})" }
+          payload = Request.build_payload(event, seq_id, debug: debug?)
+          res = @transport.send_telemetry(request_type: event.type, payload: payload)
+          logger.debug { "Telemetry sent for event `#{event.type}` (response code: #{res.code})" }
           res
         rescue => e
-          Datadog.logger.debug("Unable to send telemetry request for event `#{event.type rescue 'unknown'}`: #{e}")
-          Telemetry::Http::InternalErrorResponse.new(e)
+          logger.debug {
+            "Unable to send telemetry request for event `#{begin
+              event.type
+            rescue
+              "unknown"
+            end}`: #{e}"
+          }
+          Core::Transport::InternalErrorResponse.new(e)
         end
 
         # Initializes a Sequence object to track seq_id if not already initialized; else returns stored

data/lib/datadog/core/telemetry/event/app_client_configuration_change.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry class for the 'app-client-configuration-change' event
+        class AppClientConfigurationChange < Base
+          attr_reader :changes, :origin
+
+          def type
+            'app-client-configuration-change'
+          end
+
+          def initialize(changes, origin)
+            super()
+            @changes = changes
+            @origin = origin
+          end
+
+          def payload
+            {configuration: configuration}
+          end
+
+          def configuration
+            config = Datadog.configuration
+            seq_id = Event.configuration_sequence.next
+
+            res = @changes.map do |name, value|
+              {
+                name: name,
+                value: value,
+                origin: @origin,
+                seq_id: seq_id,
+              }
+            end
+
+            unless config.dig('appsec', 'sca_enabled').nil?
+              res << {
+                name: 'appsec.sca_enabled',
+                value: config.appsec.sca_enabled,
+                origin: 'code',
+                seq_id: seq_id,
+              }
+            end
+
+            res
+          end
+
+          def ==(other)
+            other.is_a?(AppClientConfigurationChange) && other.changes == @changes && other.origin == @origin
+          end
+
+          alias_method :eql?, :==
+
+          def hash
+            [self.class, @changes, @origin].hash
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event/app_closing.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry class for the 'app-closing' event
+        class AppClosing < Base
+          def type
+            'app-closing'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event/app_dependencies_loaded.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry class for the 'app-dependencies-loaded' event
+        class AppDependenciesLoaded < Base
+          def type
+            'app-dependencies-loaded'
+          end
+
+          def payload
+            {dependencies: dependencies}
+          end
+
+          private
+
+          def dependencies
+            Gem.loaded_specs.collect do |name, gem|
+              {
+                name: name,
+                version: gem.version.to_s,
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event/app_heartbeat.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry class for the 'app-heartbeat' event
+        class AppHeartbeat < Base
+          def type
+            'app-heartbeat'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event/app_integrations_change.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry class for the 'app-integrations-change' event
+        class AppIntegrationsChange < Base
+          def type
+            'app-integrations-change'
+          end
+
+          def payload
+            {integrations: integrations}
+          end
+
+          private
+
+          def integrations
+            instrumented_integrations = Datadog.configuration.tracing.instrumented_integrations
+            Datadog.registry.map do |integration|
+              is_instrumented = instrumented_integrations.include?(integration.name)
+
+              is_enabled = is_instrumented && integration.klass.patcher.patch_successful
+
+              version = integration.klass.class.version&.to_s
+
+              res = {
+                name: integration.name.to_s,
+                enabled: is_enabled,
+                version: version,
+                compatible: integration.klass.class.compatible?,
+                error: (patch_error(integration) if is_instrumented && !is_enabled),
+                # TODO: Track if integration is instrumented by manual configuration or by auto instrumentation
+                # auto_enabled: is_enabled && ???,
+              }
+              res.reject! { |_, v| v.nil? }
+              res
+            end
+          end
+
+          def patch_error(integration)
+            patch_error_result = integration.klass.patcher.patch_error_result
+            return patch_error_result.compact.to_s if patch_error_result
+
+            # If no error occurred during patching, but integration is still not instrumented
+            "Available?: #{integration.klass.class.available?}" \
+            ", Loaded? #{integration.klass.class.loaded?}" \
+            ", Compatible? #{integration.klass.class.compatible?}" \
+            ", Patchable? #{integration.klass.class.patchable?}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event/app_started.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry class for the 'app-started' event
+        class AppStarted < Base
+          def type
+            'app-started'
+          end
+
+          def payload
+            {
+              products: products,
+              configuration: configuration,
+              install_signature: install_signature,
+              # DEV: Not implemented yet
+              # error: error, # Start-up errors
+            }
+          end
+
+          private
+
+          def products
+            # @type var products: Hash[Symbol, Hash[Symbol, Object]]
+            products = {
+              appsec: {
+                enabled: Datadog::AppSec.enabled?,
+              },
+              profiler: {
+                enabled: Datadog::Profiling.enabled?,
+              },
+              dynamic_instrumentation: {
+                enabled: defined?(Datadog::DI) && Datadog::DI.respond_to?(:enabled?) && Datadog::DI.enabled?,
+              }
+            }
+
+            if (unsupported_reason = Datadog::Profiling.unsupported_reason)
+              products[:profiler][:error] = {
+                code: 1, # Error code. 0 if no error.
+                message: unsupported_reason,
+              }
+            end
+
+            products
+          end
+
+          TARGET_OPTIONS = %w[
+            dynamic_instrumentation.enabled
+            logger.level
+            profiling.advanced.code_provenance_enabled
+            profiling.advanced.endpoint.collection.enabled
+            profiling.enabled
+            runtime_metrics.enabled
+            tracing.analytics.enabled
+            tracing.propagation_style_extract
+            tracing.propagation_style_inject
+            tracing.enabled
+            tracing.log_injection
+            tracing.partial_flush.enabled
+            tracing.partial_flush.min_spans_threshold
+            tracing.report_hostname
+            tracing.sampling.rate_limit
+          ].freeze
+
+          # standard:disable Metrics/AbcSize
+          # standard:disable Metrics/MethodLength
+          def configuration
+            config = Datadog.configuration
+            seq_id = Event.configuration_sequence.next
+
+            list = [
+              conf_value('DD_GIT_REPOSITORY_URL', Core::Environment::Git.git_repository_url, seq_id, 'env_var'),
+              conf_value('DD_GIT_COMMIT_SHA', Core::Environment::Git.git_commit_sha, seq_id, 'env_var'),
+
+              conf_value('DD_AGENT_HOST', config.agent.host, seq_id),
+              conf_value('DD_AGENT_TRANSPORT', agent_transport(config), seq_id),
+              conf_value('DD_TRACE_SAMPLE_RATE', to_value(config.tracing.sampling.default_rate), seq_id),
+              conf_value(
+                'DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED',
+                config.tracing.contrib.global_default_service_name.enabled,
+                seq_id
+              ),
+              conf_value(
+                'DD_TRACE_PEER_SERVICE_DEFAULTS_ENABLED',
+                config.tracing.contrib.peer_service_defaults,
+                seq_id
+              ),
+            ]
+
+            peer_service_mapping_str = ''
+            unless config.tracing.contrib.peer_service_mapping.empty?
+              peer_service_mapping = config.tracing.contrib.peer_service_mapping
+              peer_service_mapping_str = peer_service_mapping.map { |key, value| "#{key}:#{value}" }.join(',')
+            end
+            list << conf_value('DD_TRACE_PEER_SERVICE_MAPPING', peer_service_mapping_str, seq_id)
+
+            # Whitelist of configuration options to send in additional payload object
+            TARGET_OPTIONS.each do |option|
+              split_option = option.split('.')
+              list << conf_value(option, to_value(config.dig(*split_option)), seq_id)
+            end
+
+            # Add some more custom additional payload values here
+            list.push(
+              conf_value('tracing.auto_instrument.enabled', !defined?(Datadog::AutoInstrument::LOADED).nil?, seq_id),
+              conf_value(
+                'tracing.writer_options.buffer_size',
+                to_value(config.tracing.writer_options[:buffer_size]),
+                seq_id
+              ),
+              conf_value(
+                'tracing.writer_options.flush_interval',
+                to_value(config.tracing.writer_options[:flush_interval]),
+                seq_id
+              ),
+              conf_value(
+                'tracing.opentelemetry.enabled',
+                !defined?(Datadog::OpenTelemetry::LOADED).nil?,
+                seq_id
+              ),
+            )
+            list << conf_value('logger.instance', config.logger.instance.class.to_s, seq_id) if config.logger.instance
+            if config.respond_to?('appsec')
+              list << conf_value('appsec.enabled', config.dig('appsec', 'enabled'), seq_id)
+              list << conf_value('appsec.sca_enabled', config.dig('appsec', 'sca_enabled'), seq_id)
+            end
+            list << conf_value('ci.enabled', config.dig('ci', 'enabled'), seq_id) if config.respond_to?('ci')
+
+            list.reject! { |entry| entry[:value].nil? }
+            list
+          end
+          # standard:enable Metrics/AbcSize
+          # standard:enable Metrics/MethodLength
+
+          def agent_transport(config)
+            adapter = Core::Configuration::AgentSettingsResolver.call(config).adapter
+            if adapter == Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
+              'UDS'
+            else
+              'TCP'
+            end
+          end
+
+          def conf_value(name, value, seq_id, origin = 'code')
+            {
+              name: name,
+              value: value,
+              origin: origin,
+              seq_id: seq_id,
+            }
+          end
+
+          def to_value(value)
+            # TODO: Add float if telemetry starts accepting it
+            case value
+            when Integer, String, true, false, nil
+              value
+            else
+              value.to_s
+            end
+          end
+
+          def install_signature
+            config = Datadog.configuration
+            {
+              install_id: config.dig('telemetry', 'install_id'),
+              install_type: config.dig('telemetry', 'install_type'),
+              install_time: config.dig('telemetry', 'install_time'),
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event/base.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Base class for all Telemetry V2 events.
+        class Base
+          # The type of the event.
+          # It must be one of the stings defined in the Telemetry V2
+          # specification for event names.
+          def type
+            raise NotImplementedError, 'Must be implemented by subclass'
+          end
+
+          # The JSON payload for the event.
+          def payload
+            {}
+          end
+
+          # Override equality to allow for deduplication
+          # The basic implementation is to check if the other object is an instance of the same class.
+          # This works for events that have no attributes.
+          # For events with attributes, you should override this method to compare the attributes.
+          def ==(other)
+            other.is_a?(self.class)
+          end
+
+          # @see #==
+          alias_method :eql?, :==
+
+          # @see #==
+          def hash
+            self.class.hash
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/event/distributions.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative 'generate_metrics'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Event
+        # Telemetry class for the 'distributions' event
+        class Distributions < GenerateMetrics
+          def type
+            'distributions'
+          end
+        end
+      end
+    end
+  end
+end