datadog 2.1.0 → 2.3.0

data/lib/datadog/core/telemetry/emitter.rb

@@ -16,22 +16,20 @@ module Datadog
 
         # @param http_transport [Datadog::Core::Telemetry::Http::Transport] Transport object that can be used to send
         #   telemetry requests via the agent
-        def initialize(http_transport: Datadog::Core::Telemetry::Http::Transport.new)
+        def initialize(http_transport:)
           @http_transport = http_transport
         end
 
         # Retrieves and emits a TelemetryRequest object based on the request type specified
         def request(event)
-          begin
-            seq_id = self.class.sequence.next
-            payload = Request.build_payload(event, seq_id)
-            res = @http_transport.request(request_type: event.type, payload: payload.to_json)
-            Datadog.logger.debug { "Telemetry sent for event `#{event.type}` (code: #{res.code.inspect})" }
-            res
-          rescue => e
-            Datadog.logger.debug("Unable to send telemetry request for event `#{event.type rescue 'unknown'}`: #{e}")
-            Telemetry::Http::InternalErrorResponse.new(e)
-          end
+          seq_id = self.class.sequence.next
+          payload = Request.build_payload(event, seq_id)
+          res = @http_transport.request(request_type: event.type, payload: payload.to_json)
+          Datadog.logger.debug { "Telemetry sent for event `#{event.type}` (code: #{res.code.inspect})" }
+          res
+        rescue => e
+          Datadog.logger.debug("Unable to send telemetry request for event `#{event.type rescue 'unknown'}`: #{e}")
+          Telemetry::Http::InternalErrorResponse.new(e)
         end
 
         # Initializes a Sequence object to track seq_id if not already initialized; else returns stored

data/lib/datadog/core/telemetry/event.rb

@@ -3,17 +3,27 @@
 module Datadog
   module Core
     module Telemetry
+      # Collection of telemetry events
       class Event
+        extend Core::Utils::Forking
+
+        # returns sequence that increments every time the configuration changes
+        def self.configuration_sequence
+          after_fork! { @sequence = Datadog::Core::Utils::Sequence.new(1) }
+          @sequence ||= Datadog::Core::Utils::Sequence.new(1)
+        end
+
         # Base class for all Telemetry V2 events.
         class Base
           # The type of the event.
           # It must be one of the stings defined in the Telemetry V2
           # specification for event names.
-          def type; end
+          def type
+            raise NotImplementedError, 'Must be implemented by subclass'
+          end
 
           # The JSON payload for the event.
-          # @param seq_id [Integer] The sequence ID for the event.
-          def payload(seq_id)
+          def payload
             {}
           end
         end
@@ -24,8 +34,7 @@ module Datadog
             'app-started'
           end
 
-          def payload(seq_id)
-            @seq_id = seq_id
+          def payload
             {
               products: products,
               configuration: configuration,
@@ -38,6 +47,7 @@ module Datadog
           private
 
           def products
+            # @type var products: Hash[Symbol, Hash[Symbol, Object]]
             products = {
               appsec: {
                 enabled: Datadog::AppSec.enabled?,
@@ -79,16 +89,19 @@ module Datadog
           ].freeze
 
           # rubocop:disable Metrics/AbcSize
+          # rubocop:disable Metrics/MethodLength
           def configuration
             config = Datadog.configuration
+            seq_id = Event.configuration_sequence.next
 
             list = [
-              conf_value('DD_AGENT_HOST', config.agent.host),
-              conf_value('DD_AGENT_TRANSPORT', agent_transport(config)),
-              conf_value('DD_TRACE_SAMPLE_RATE', to_value(config.tracing.sampling.default_rate)),
+              conf_value('DD_AGENT_HOST', config.agent.host, seq_id),
+              conf_value('DD_AGENT_TRANSPORT', agent_transport(config), seq_id),
+              conf_value('DD_TRACE_SAMPLE_RATE', to_value(config.tracing.sampling.default_rate), seq_id),
               conf_value(
                 'DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED',
-                config.tracing.contrib.global_default_service_name.enabled
+                config.tracing.contrib.global_default_service_name.enabled,
+                seq_id
               ),
             ]
 
@@ -97,32 +110,45 @@ module Datadog
               peer_service_mapping = config.tracing.contrib.peer_service_mapping
               peer_service_mapping_str = peer_service_mapping.map { |key, value| "#{key}:#{value}" }.join(',')
             end
-            list << conf_value('DD_TRACE_PEER_SERVICE_MAPPING', peer_service_mapping_str)
+            list << conf_value('DD_TRACE_PEER_SERVICE_MAPPING', peer_service_mapping_str, seq_id)
 
             # Whitelist of configuration options to send in additional payload object
             TARGET_OPTIONS.each do |option|
               split_option = option.split('.')
-              list << conf_value(option, to_value(config.dig(*split_option)))
+              list << conf_value(option, to_value(config.dig(*split_option)), seq_id)
             end
 
             # Add some more custom additional payload values here
             list.push(
-              conf_value('tracing.auto_instrument.enabled', !defined?(Datadog::AutoInstrument::LOADED).nil?),
-              conf_value('tracing.writer_options.buffer_size', to_value(config.tracing.writer_options[:buffer_size])),
-              conf_value('tracing.writer_options.flush_interval', to_value(config.tracing.writer_options[:flush_interval])),
-              conf_value('tracing.opentelemetry.enabled', !defined?(Datadog::OpenTelemetry::LOADED).nil?),
+              conf_value('tracing.auto_instrument.enabled', !defined?(Datadog::AutoInstrument::LOADED).nil?, seq_id),
+              conf_value(
+                'tracing.writer_options.buffer_size',
+                to_value(config.tracing.writer_options[:buffer_size]),
+                seq_id
+              ),
+              conf_value(
+                'tracing.writer_options.flush_interval',
+                to_value(config.tracing.writer_options[:flush_interval]),
+                seq_id
+              ),
+              conf_value(
+                'tracing.opentelemetry.enabled',
+                !defined?(Datadog::OpenTelemetry::LOADED).nil?,
+                seq_id
+              ),
             )
-            list << conf_value('logger.instance', config.logger.instance.class.to_s) if config.logger.instance
+            list << conf_value('logger.instance', config.logger.instance.class.to_s, seq_id) if config.logger.instance
             if config.respond_to?('appsec')
-              list << conf_value('appsec.enabled', config.dig('appsec', 'enabled'))
-              list << conf_value('appsec.sca_enabled', config.dig('appsec', 'sca_enabled'))
+              list << conf_value('appsec.enabled', config.dig('appsec', 'enabled'), seq_id)
+              list << conf_value('appsec.sca_enabled', config.dig('appsec', 'sca_enabled'), seq_id)
             end
-            list << conf_value('ci.enabled', config.dig('ci', 'enabled')) if config.respond_to?('ci')
+            list << conf_value('ci.enabled', config.dig('ci', 'enabled'), seq_id) if config.respond_to?('ci')
 
             list.reject! { |entry| entry[:value].nil? }
             list
           end
           # rubocop:enable Metrics/AbcSize
+          # rubocop:enable Metrics/MethodLength
 
           def agent_transport(config)
             adapter = Core::Configuration::AgentSettingsResolver.call(config).adapter
@@ -133,12 +159,12 @@ module Datadog
             end
           end
 
-          def conf_value(name, value, origin = 'code')
+          def conf_value(name, value, seq_id, origin = 'code')
             {
               name: name,
               value: value,
               origin: origin,
-              seq_id: @seq_id,
+              seq_id: seq_id,
             }
           end
 
@@ -168,7 +194,7 @@ module Datadog
             'app-dependencies-loaded'
           end
 
-          def payload(seq_id)
+          def payload
             { dependencies: dependencies }
           end
 
@@ -191,7 +217,7 @@ module Datadog
             'app-integrations-change'
           end
 
-          def payload(seq_id)
+          def payload
             { integrations: integrations }
           end
 
@@ -244,18 +270,20 @@ module Datadog
             @origin = origin
           end
 
-          def payload(seq_id)
-            { configuration: configuration(seq_id) }
+          def payload
+            { configuration: configuration }
           end
 
-          def configuration(seq_id)
+          def configuration
             config = Datadog.configuration
+            seq_id = Event.configuration_sequence.next
 
             res = @changes.map do |name, value|
               {
                 name: name,
                 value: value,
                 origin: @origin,
+                seq_id: seq_id,
               }
             end
 
@@ -285,6 +313,84 @@ module Datadog
             'app-closing'
           end
         end
+
+        # Telemetry class for the 'generate-metrics' event
+        class GenerateMetrics < Base
+          def type
+            'generate-metrics'
+          end
+
+          def initialize(namespace, metric_series)
+            super()
+            @namespace = namespace
+            @metric_series = metric_series
+          end
+
+          def payload
+            {
+              namespace: @namespace,
+              series: @metric_series.map(&:to_h)
+            }
+          end
+        end
+
+        # Telemetry class for the 'logs' event
+        class Log < Base
+          LEVELS = {
+            error: 'ERROR',
+            debug: 'DEBUG',
+            warn: 'WARN',
+          }.freeze
+
+          def type
+            'logs'
+          end
+
+          def initialize(message:, level:)
+            super()
+            @message = message
+            @level = LEVELS.fetch(level) { |k| raise ArgumentError, "Invalid log level :#{k}" }
+          end
+
+          def payload
+            {
+              logs: [{
+                message: @message,
+                level: @level,
+                # More optional fields to be added here...
+              }]
+            }
+          end
+        end
+
+        # Telemetry class for the 'distributions' event
+        class Distributions < GenerateMetrics
+          def type
+            'distributions'
+          end
+        end
+
+        # Telemetry class for the 'message-batch' event
+        class MessageBatch
+          attr_reader :events
+
+          def type
+            'message-batch'
+          end
+
+          def initialize(events)
+            @events = events
+          end
+
+          def payload
+            @events.map do |event|
+              {
+                request_type: event.type,
+                payload: event.payload,
+              }
+            end
+          end
+        end
       end
     end
   end

data/lib/datadog/core/telemetry/ext.rb

@@ -5,11 +5,14 @@ module Datadog
     module Telemetry
       module Ext
         ENV_ENABLED = 'DD_INSTRUMENTATION_TELEMETRY_ENABLED'
+        ENV_METRICS_ENABLED = 'DD_TELEMETRY_METRICS_ENABLED'
         ENV_HEARTBEAT_INTERVAL = 'DD_TELEMETRY_HEARTBEAT_INTERVAL'
+        ENV_METRICS_AGGREGATION_INTERVAL = 'DD_TELEMETRY_METRICS_AGGREGATION_INTERVAL'
         ENV_DEPENDENCY_COLLECTION = 'DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED'
         ENV_INSTALL_ID = 'DD_INSTRUMENTATION_INSTALL_ID'
         ENV_INSTALL_TYPE = 'DD_INSTRUMENTATION_INSTALL_TYPE'
         ENV_INSTALL_TIME = 'DD_INSTRUMENTATION_INSTALL_TIME'
+        ENV_AGENTLESS_URL_OVERRIDE = 'DD_TELEMETRY_AGENTLESS_URL'
       end
     end
   end

data/lib/datadog/core/telemetry/http/adapters/net.rb

@@ -15,7 +15,7 @@ module Datadog
               :timeout,
               :ssl
 
-            DEFAULT_TIMEOUT = 30
+            DEFAULT_TIMEOUT = 2
 
             def initialize(hostname:, port: nil, timeout: DEFAULT_TIMEOUT, ssl: true)
               @hostname = hostname
@@ -34,19 +34,17 @@ module Datadog
             end
 
             def post(env)
-              begin
-                post = ::Net::HTTP::Post.new(env.path, env.headers)
-                post.body = env.body
-
-                http_response = open do |http|
-                  http.request(post)
-                end
-
-                Response.new(http_response)
-              rescue StandardError => e
-                Datadog.logger.debug("Unable to send telemetry event to agent: #{e}")
-                Telemetry::Http::InternalErrorResponse.new(e)
+              post = ::Net::HTTP::Post.new(env.path, env.headers)
+              post.body = env.body
+
+              http_response = open do |http|
+                http.request(post)
               end
+
+              Response.new(http_response)
+            rescue StandardError => e
+              Datadog.logger.debug("Unable to send telemetry event to agent: #{e}")
+              Telemetry::Http::InternalErrorResponse.new(e)
             end
 
             # Data structure for an HTTP Response

data/lib/datadog/core/telemetry/http/ext.rb

@@ -17,7 +17,10 @@ module Datadog
           CONTENT_TYPE_APPLICATION_JSON = 'application/json'
           API_VERSION = 'v2'
 
+          AGENTLESS_HOST_PREFIX = 'instrumentation-telemetry-intake'
+
           AGENT_ENDPOINT = '/telemetry/proxy/api/v2/apmtelemetry'
+          AGENTLESS_ENDPOINT = '/api/v2/apmtelemetry'
         end
       end
     end

data/lib/datadog/core/telemetry/http/transport.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../configuration/settings'
+require_relative '../../environment/ext'
 require_relative '../../transport/ext'
 require_relative 'env'
 require_relative 'ext'
@@ -13,18 +14,42 @@ module Datadog
         # Class to send telemetry data to Telemetry API
         # Currently only supports the HTTP protocol.
         class Transport
+          def self.build_agent_transport(agent_settings)
+            Transport.new(
+              host: agent_settings.hostname,
+              port: agent_settings.port,
+              path: Http::Ext::AGENT_ENDPOINT
+            )
+          end
+
+          def self.build_agentless_transport(api_key:, dd_site:, url_override: nil)
+            url = url_override || "https://#{Http::Ext::AGENTLESS_HOST_PREFIX}.#{dd_site}:443"
+
+            uri = URI.parse(url)
+            raise "Invalid agentless mode URL: #{url}" if uri.host.nil?
+
+            Transport.new(
+              host: uri.host,
+              port: uri.port || 80,
+              path: Http::Ext::AGENTLESS_ENDPOINT,
+              ssl: uri.scheme == 'https' || uri.port == 443,
+              api_key: api_key
+            )
+          end
+
           attr_reader \
             :host,
             :port,
             :ssl,
-            :path
-
-          def initialize
-            agent_settings = Configuration::AgentSettingsResolver.call(Datadog.configuration)
-            @host = agent_settings.hostname
-            @port = agent_settings.port
-            @ssl = false
-            @path = Http::Ext::AGENT_ENDPOINT
+            :path,
+            :api_key
+
+          def initialize(host:, port:, path:, ssl: false, api_key: nil)
+            @host = host
+            @port = port
+            @ssl = ssl
+            @path = path
+            @api_key = api_key
           end
 
           def request(request_type:, payload:)
@@ -38,7 +63,7 @@ module Datadog
           private
 
           def headers(request_type:, api_version: Http::Ext::API_VERSION)
-            {
+            result = {
               Core::Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST => '1',
               Ext::HEADER_CONTENT_TYPE => Http::Ext::CONTENT_TYPE_APPLICATION_JSON,
               Ext::HEADER_DD_TELEMETRY_API_VERSION => api_version,
@@ -49,6 +74,10 @@ module Datadog
               # Enable debug mode for telemetry
               # HEADER_TELEMETRY_DEBUG_ENABLED => 'true',
             }
+
+            result[Ext::HEADER_DD_API_KEY] = api_key unless api_key.nil?
+
+            result
           end
 
           def adapter

data/lib/datadog/core/telemetry/logging.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative 'event'
+
+module Datadog
+  module Core
+    module Telemetry
+      # Logging interface for sending telemetry logs.
+      #
+      # Reporting internal error so that we can fix them.
+      # IMPORTANT: Make sure to not log any sensitive information.
+      module Logging
+        module_function
+
+        def report(exception, level:)
+          # Annoymous exceptions to be logged as <Class:0x00007f8b1c0b3b40>
+          message = exception.class.name || exception.class.inspect
+
+          event = Event::Log.new(
+            message: message,
+            level: level
+          )
+
+          if (telemetry = Datadog.send(:components).telemetry)
+            telemetry.log!(event)
+          else
+            Datadog.logger.debug do
+              "Attempting to send telemetry log when telemetry component is not ready: #{message}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/metric.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Telemetry
+      # Telemetry metrics data model (internal Datadog metrics for client libraries)
+      module Metric
+        # Base class for all metric types
+        class Base
+          attr_reader :name, :tags, :values, :common
+
+          # @param name [String] metric name
+          # @param tags [Array<String>|Hash{String=>String}] metric tags as hash or array of "tag:val" strings
+          # @param common [Boolean] true if the metric is common for all languages, false for Ruby-specific metric
+          def initialize(name, tags: {}, common: true)
+            @name = name
+            @values = []
+            @tags = tags_to_array(tags)
+            @common = common
+          end
+
+          def id
+            @id ||= "#{type}::#{name}::#{tags.join(',')}"
+          end
+
+          def track(value)
+            raise NotImplementedError, 'method must be implemented in subclasses'
+          end
+
+          def type
+            raise NotImplementedError, 'method must be implemented in subclasses'
+          end
+
+          def to_h
+            {
+              metric: name,
+              points: values,
+              type: type,
+              tags: tags,
+              common: common
+            }
+          end
+
+          private
+
+          def tags_to_array(tags)
+            return tags if tags.is_a?(Array)
+
+            tags.map { |k, v| "#{k}:#{v}" }
+          end
+        end
+
+        # Base class for metrics that require aggregation interval
+        class IntervalMetric < Base
+          attr_reader :interval
+
+          # @param name [String] metric name
+          # @param tags [Array<String>|Hash{String=>String}] metric tags as hash of array of "tag:val" strings
+          # @param common [Boolean] true if the metric is common for all languages, false for Ruby-specific metric
+          # @param interval [Integer] metrics aggregation interval in seconds
+          def initialize(name, interval:, tags: {}, common: true)
+            raise ArgumentError, 'interval must be a positive number' if interval.nil? || interval <= 0
+
+            super(name, tags: tags, common: common)
+
+            @interval = interval
+          end
+
+          def to_h
+            res = super
+            res[:interval] = interval
+            res
+          end
+        end
+
+        # Count metric adds up all the submitted values in a time interval. This would be suitable for a
+        # metric tracking the number of website hits, for instance.
+        class Count < Base
+          TYPE = 'count'
+
+          def type
+            TYPE
+          end
+
+          def track(value)
+            value = value.to_i
+
+            if values.empty?
+              values << [Time.now.to_i, value]
+            else
+              values[0][0] = Time.now.to_i
+              values[0][1] += value
+            end
+            nil
+          end
+        end
+
+        # A gauge type takes the last value reported during the interval. This type would make sense for tracking RAM or
+        # CPU usage, where taking the last value provides a representative picture of the host’s behavior during the time
+        # interval.
+        class Gauge < IntervalMetric
+          TYPE = 'gauge'
+
+          def type
+            TYPE
+          end
+
+          def track(value)
+            if values.empty?
+              values << [Time.now.to_i, value]
+            else
+              values[0][0] = Time.now.to_i
+              values[0][1] = value
+            end
+            nil
+          end
+        end
+
+        # The rate type takes the count and divides it by the length of the time interval. This is useful if you’re
+        # interested in the number of hits per second.
+        class Rate < IntervalMetric
+          TYPE = 'rate'
+
+          def initialize(name, interval:, tags: {}, common: true)
+            super
+
+            @value = 0.0
+          end
+
+          def type
+            TYPE
+          end
+
+          def track(value = 1.0)
+            @value += value
+            @values = [[Time.now.to_i, @value / interval]]
+            nil
+          end
+        end
+
+        # Distribution metric represents the global statistical distribution of a set of values.
+        class Distribution < Base
+          TYPE = 'distributions'
+
+          def type
+            TYPE
+          end
+
+          def track(value)
+            values << value
+            nil
+          end
+
+          # distribution metric data does not have type field
+          def to_h
+            {
+              metric: name,
+              points: values,
+              tags: tags,
+              common: common
+            }
+          end
+        end
+      end
+    end
+  end
+end