datadog 2.7.1 → 2.17.0

data/lib/datadog/core/telemetry/worker.rb

@@ -17,14 +17,13 @@ module Datadog
         DEFAULT_BUFFER_MAX_SIZE = 1000
         APP_STARTED_EVENT_RETRIES = 10
 
-        TELEMETRY_STARTED_ONCE = Utils::OnlyOnceSuccessful.new(APP_STARTED_EVENT_RETRIES)
-
         def initialize(
           heartbeat_interval_seconds:,
           metrics_aggregation_interval_seconds:,
           emitter:,
           metrics_manager:,
           dependency_collection:,
+          logger:,
           enabled: true,
           shutdown_timeout: Workers::Polling::DEFAULT_SHUTDOWN_TIMEOUT,
           buffer_size: DEFAULT_BUFFER_MAX_SIZE
@@ -32,6 +31,7 @@ module Datadog
           @emitter = emitter
           @metrics_manager = metrics_manager
           @dependency_collection = dependency_collection
+          @logger = logger
 
           @ticks_per_heartbeat = (heartbeat_interval_seconds / metrics_aggregation_interval_seconds).to_i
           @current_ticks = 0
@@ -46,12 +46,25 @@ module Datadog
           @buffer_size = buffer_size
 
           self.buffer = buffer_klass.new(@buffer_size)
+
+          @initial_event_once = Utils::OnlyOnceSuccessful.new(APP_STARTED_EVENT_RETRIES)
         end
 
-        def start
+        attr_reader :logger
+        attr_reader :initial_event_once
+        attr_reader :initial_event
+
+        # Returns true if worker thread is successfully started,
+        # false if worker thread was not started but telemetry is enabled,
+        # nil if telemetry is disabled.
+        def start(initial_event)
           return if !enabled? || forked?
 
+          @initial_event = initial_event
+
           # starts async worker
+          # perform should return true if thread was actually started,
+          # false otherwise
           perform
         end
 
@@ -61,18 +74,60 @@ module Datadog
           super
         end
 
+        # Returns true if event was enqueued, nil if not.
+        # While returning false may seem more reasonable, the only reason
+        # for not enqueueing event (presently) is that telemetry is disabled
+        # altogether, and in this case other methods return nil.
         def enqueue(event)
           return if !enabled? || forked?
 
           buffer.push(event)
+          true
+        end
+
+        def sent_initial_event?
+          initial_event_once.success?
+        end
+
+        def failed_initial_event?
+          initial_event_once.failed?
         end
 
-        def sent_started_event?
-          TELEMETRY_STARTED_ONCE.success?
+        def need_initial_event?
+          !sent_initial_event? && !failed_initial_event?
         end
 
-        def failed_to_start?
-          TELEMETRY_STARTED_ONCE.failed?
+        # Wait for the worker to send out all events that have already
+        # been queued, up to 15 seconds. Returns whether all events have
+        # been flushed.
+        #
+        # @api private
+        def flush
+          return true unless enabled? || !run_loop?
+
+          started = Utils::Time.get_time
+          loop do
+            # The AppStarted event is triggered by the worker itself,
+            # from the worker thread. As such the main thread has no way
+            # to delay itself until that event is queued and we need some
+            # way to wait until that event is sent out to assert on it in
+            # the test suite. Check the run once flag which *should*
+            # indicate the event has been queued (at which point our queue
+            # depth check should waint until it's sent).
+            # This is still a hack because the flag can be overridden
+            # either way with or without the event being sent out.
+            # Note that if the AppStarted sending fails, this check
+            # will return false and flushing will be blocked until the
+            # 15 second timeout.
+            # Note that the first wait interval between telemetry event
+            # sending is 10 seconds, the timeout needs to be strictly
+            # greater than that.
+            return true if buffer.empty? && !in_iteration? && sent_initial_event?
+
+            sleep 0.5
+
+            return false if Utils::Time.get_time - started > 15
+          end
         end
 
         private
@@ -80,11 +135,26 @@ module Datadog
         def perform(*events)
           return if !enabled? || forked?
 
-          started! unless sent_started_event?
+          if need_initial_event?
+            started!
+            unless sent_initial_event?
+              # We still haven't succeeded in sending the started event,
+              # which will make flush_events do nothing - but the events
+              # given to us as the parameter have already been removed
+              # from the queue.
+              # Put the events back to the front of the queue to not
+              # lose them.
+              buffer.unshift(*events)
+              return
+            end
+          end
 
           metric_events = @metrics_manager.flush!
           events = [] if events.nil?
-          flush_events(events + metric_events)
+          events += metric_events
+          if events.any?
+            flush_events(events)
+          end
 
           @current_ticks += 1
           return if @current_ticks < @ticks_per_heartbeat
@@ -94,15 +164,14 @@ module Datadog
         end
 
         def flush_events(events)
-          return if events.empty?
-          return if !enabled? || !sent_started_event?
+          events = deduplicate_logs(events)
 
-          Datadog.logger.debug { "Sending #{events&.count} telemetry events" }
+          logger.debug { "Sending #{events&.count} telemetry events" }
           send_event(Event::MessageBatch.new(events))
         end
 
         def heartbeat!
-          return if !enabled? || !sent_started_event?
+          return if !enabled? || !sent_initial_event?
 
           send_event(Event::AppHeartbeat.new)
         end
@@ -110,26 +179,29 @@ module Datadog
         def started!
           return unless enabled?
 
-          if failed_to_start?
-            Datadog.logger.debug('Telemetry app-started event exhausted retries, disabling telemetry worker')
-            disable!
-            return
-          end
-
-          TELEMETRY_STARTED_ONCE.run do
-            res = send_event(Event::AppStarted.new)
+          initial_event_once.run do
+            res = send_event(initial_event)
 
             if res.ok?
-              Datadog.logger.debug('Telemetry app-started event is successfully sent')
+              logger.debug { "Telemetry initial event (#{initial_event.type}) is successfully sent" }
 
-              send_event(Event::AppDependenciesLoaded.new) if @dependency_collection
+              # TODO Dependencies loaded event should probably check for new
+              # dependencies and send the new ones.
+              # System tests demand only one instance of this event per
+              # dependency.
+              send_event(Event::AppDependenciesLoaded.new) if @dependency_collection && initial_event.class.eql?(Telemetry::Event::AppStarted) # standard:disable Style/ClassEqualityComparison:
 
               true
             else
-              Datadog.logger.debug('Error sending telemetry app-started event, retry after heartbeat interval...')
+              logger.debug("Error sending telemetry initial event (#{initial_event.type}), retry after heartbeat interval...")
               false
             end
           end
+
+          if failed_initial_event?
+            logger.debug { "Telemetry initial event (#{initial_event.type}) exhausted retries, disabling telemetry worker" }
+            disable!
+          end
         end
 
         def send_event(event)
@@ -164,9 +236,40 @@ module Datadog
         def disable_on_not_found!(response)
           return unless response.not_found?
 
-          Datadog.logger.debug('Agent does not support telemetry; disabling future telemetry events.')
+          logger.debug('Agent does not support telemetry; disabling future telemetry events.')
           disable!
         end
+
+        # Deduplicate logs by counting the number of repeated occurrences of the same log
+        # entry and replacing them with a single entry with the calculated `count` value.
+        # Non-log events are unchanged.
+        def deduplicate_logs(events)
+          return events if events.empty?
+
+          all_logs = []
+          other_events = events.reject do |event|
+            if event.is_a?(Event::Log)
+              all_logs << event
+              true
+            else
+              false
+            end
+          end
+
+          return events if all_logs.empty?
+
+          uniq_logs = all_logs.group_by(&:itself).map do |_, logs|
+            log = logs.first
+            if logs.size > 1
+              # New log event with a count of repeated occurrences
+              Event::Log.new(message: log.message, level: log.level, stack_trace: log.stack_trace, count: logs.size)
+            else
+              log
+            end
+          end
+
+          other_events + uniq_logs
+        end
       end
     end
   end

data/lib/datadog/core/transport/http/adapters/test.rb

@@ -38,7 +38,8 @@ module Datadog
               @status = status
             end
 
-            def url; end
+            def url
+            end
 
             # Response for test adapter
             class Response

data/lib/datadog/core/transport/http/adapters/unix_socket.rb

@@ -49,7 +49,7 @@ module Datadog
 
             # Re-implements Net:HTTP with underlying Unix socket
             class HTTP < ::Net::HTTP
-              DEFAULT_TIMEOUT = 1
+              DEFAULT_TIMEOUT = 30
 
               attr_reader \
                 :filepath, # DEV(1.0): Rename to `uds_path`

data/lib/datadog/{tracing → core}/transport/http/api/instance.rb

@@ -1,12 +1,29 @@
 # frozen_string_literal: true
 
 module Datadog
-  module Tracing
+  module Core
     module Transport
       module HTTP
         module API
           # An API configured with adapter and routes
           class Instance
+            # Raised when an endpoint is invoked on an API that is not the
+            # of expected API class for that endpoint.
+            class EndpointNotSupportedError < StandardError
+              attr_reader :spec, :endpoint_name
+
+              def initialize(endpoint_name, spec)
+                @spec = spec
+                @endpoint_name = endpoint_name
+
+                super(message)
+              end
+
+              def message
+                "#{endpoint_name} not supported for this API!"
+              end
+            end
+
             attr_reader \
               :adapter,
               :headers,

data/lib/datadog/core/transport/http/api/spec.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Transport
+      module HTTP
+        module API
+          # Specification for an HTTP API
+          # Defines behaviors without specific configuration details.
+          class Spec
+            # Raised when an endpoint is invoked on an API that did not
+            # define that endpoint.
+            class EndpointNotDefinedError < StandardError
+              attr_reader :spec, :endpoint_name
+
+              def initialize(endpoint_name, spec)
+                @spec = spec
+                @endpoint_name = endpoint_name
+
+                super(message)
+              end
+
+              def message
+                "No #{endpoint_name} endpoint is defined for API specification!"
+              end
+            end
+
+            def initialize
+              yield(self) if block_given?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/{tracing → core}/transport/http/builder.rb

@@ -1,13 +1,11 @@
 # frozen_string_literal: true
 
-require_relative '../../../core/configuration/agent_settings_resolver'
-require_relative '../../../core/transport/http/adapters/registry'
-require_relative '../../../core/transport/http/api/map'
-require_relative 'api/instance'
-require_relative 'client'
+require_relative '../../configuration/agent_settings_resolver'
+require_relative 'adapters/registry'
+require_relative 'api/map'
 
 module Datadog
-  module Tracing
+  module Core
     module Transport
       module HTTP
         # Builds new instances of Transport::HTTP::Client
@@ -15,13 +13,15 @@ module Datadog
           REGISTRY = Datadog::Core::Transport::HTTP::Adapters::Registry.new
 
           attr_reader \
+            :api_instance_class,
             :apis,
             :api_options,
             :default_adapter,
             :default_api,
-            :default_headers
+            :default_headers,
+            :logger
 
-          def initialize
+          def initialize(api_instance_class:, logger: Datadog.logger)
             # Global settings
             @default_adapter = nil
             @default_headers = {}
@@ -33,24 +33,27 @@ module Datadog
             # API settings
             @api_options = {}
 
+            @api_instance_class = api_instance_class
+            @logger = logger
+
             yield(self) if block_given?
           end
 
           def adapter(config, *args, **kwargs)
             @default_adapter = case config
-                               when Core::Configuration::AgentSettingsResolver::AgentSettings
-                                 registry_klass = REGISTRY.get(config.adapter)
-                                 raise UnknownAdapterError, config.adapter if registry_klass.nil?
-
-                                 registry_klass.build(config)
-                               when Symbol
-                                 registry_klass = REGISTRY.get(config)
-                                 raise UnknownAdapterError, config if registry_klass.nil?
-
-                                 registry_klass.new(*args, **kwargs)
-                               else
-                                 config
-                               end
+            when Core::Configuration::AgentSettingsResolver::AgentSettings
+              registry_klass = REGISTRY.get(config.adapter)
+              raise UnknownAdapterError, config.adapter if registry_klass.nil?
+
+              registry_klass.build(config)
+            when Symbol
+              registry_klass = REGISTRY.get(config)
+              raise UnknownAdapterError, config if registry_klass.nil?
+
+              registry_klass.new(*args, **kwargs)
+            else
+              config
+            end
           end
 
           def headers(values = {})
@@ -82,11 +85,10 @@ module Datadog
             @default_api = key
           end
 
-          def to_transport
+          def to_transport(klass)
             raise NoDefaultApiError if @default_api.nil?
 
-            # DEV: Should not be specific to traces
-            Transport::Traces::Transport.new(to_api_instances, @default_api)
+            klass.new(to_api_instances, @default_api, logger: logger)
           end
 
           def to_api_instances
@@ -117,28 +119,48 @@ module Datadog
             end
           end
 
-          def api_instance_class
-            API::Instance
-          end
-
           # Raised when the API key does not match known APIs.
           class UnknownApiError < StandardError
+            attr_reader :key
+
             def initialize(key)
-              super("Unknown transport API '#{key}'!")
+              super()
+
+              @key = key
+            end
+
+            def message
+              "Unknown transport API '#{key}'!"
             end
           end
 
           # Raised when the identifier cannot be matched to an adapter.
           class UnknownAdapterError < StandardError
+            attr_reader :type
+
             def initialize(type)
-              super("Unknown transport adapter '#{type}'!")
+              super()
+
+              @type = type
+            end
+
+            def message
+              "Unknown transport adapter '#{type}'!"
             end
           end
 
           # Raised when an adapter cannot be resolved for an API instance.
           class NoAdapterForApiError < StandardError
+            attr_reader :key
+
             def initialize(key)
-              super("No adapter resolved for transport API '#{key}'!")
+              super()
+
+              @key = key
+            end
+
+            def message
+              "No adapter resolved for transport API '#{key}'!"
             end
           end
 

data/lib/datadog/core/transport/http.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require_relative 'http/builder'
+require_relative 'http/adapters/net'
+require_relative 'http/adapters/unix_socket'
+require_relative 'http/adapters/test'
+
+module Datadog
+  module Core
+    module Transport
+      # HTTP transport
+      module HTTP
+        # Add adapters to registry
+        Builder::REGISTRY.set(
+          Transport::HTTP::Adapters::Net,
+          Core::Configuration::Ext::Agent::HTTP::ADAPTER
+        )
+        Builder::REGISTRY.set(
+          Transport::HTTP::Adapters::Test,
+          Transport::Ext::Test::ADAPTER
+        )
+        Builder::REGISTRY.set(
+          Transport::HTTP::Adapters::UnixSocket,
+          Transport::Ext::UnixSocket::ADAPTER
+        )
+
+        module_function
+
+        # Helper function that delegates to Builder.new
+        # but is under HTTP namespace so that client code requires this file
+        # to get the adapters configured, and not the builder directly.
+        def build(api_instance_class:, agent_settings:, logger: Datadog.logger, api_version: nil, headers: nil, &block)
+          Builder.new(api_instance_class: api_instance_class, logger: logger) do |transport|
+            transport.adapter(agent_settings)
+            transport.headers(default_headers)
+
+            # The caller must define APIs before we set the default API.
+            yield transport
+
+            # Apply any settings given by options
+            transport.default_api = api_version if api_version
+            transport.headers(headers) if headers
+          end
+        end
+
+        def default_headers
+          {
+            Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_TOP_LEVEL => '1',
+            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG =>
+              Datadog::Core::Environment::Ext::LANG,
+            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_VERSION =>
+              Datadog::Core::Environment::Ext::LANG_VERSION,
+            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER =>
+              Datadog::Core::Environment::Ext::LANG_INTERPRETER,
+            Datadog::Core::Transport::Ext::HTTP::HEADER_META_LANG_INTERPRETER_VENDOR =>
+              Core::Environment::Ext::LANG_ENGINE,
+            Datadog::Core::Transport::Ext::HTTP::HEADER_META_TRACER_VERSION =>
+              Datadog::Core::Environment::Ext::GEM_DATADOG_VERSION
+          }.tap do |headers|
+            # Add container ID, if present.
+            if (container_id = Datadog::Core::Environment::Container.container_id)
+              headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CONTAINER_ID] = container_id
+            end
+            # TODO: inject configuration rather than reading from global here
+            unless Datadog.configuration.apm.tracing.enabled
+              # Sending this header to the agent will disable metrics computation (and billing) on the agent side
+              # by pretending it has already been done on the library side.
+              headers[Datadog::Core::Transport::Ext::HTTP::HEADER_CLIENT_COMPUTED_STATS] = 'yes'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/transport/response.rb

@@ -55,6 +55,10 @@ module Datadog
           true
         end
 
+        def to_s
+          "#{super}, error_type:#{error.class} error:#{error}"
+        end
+
         def inspect
           "#{super}, error_type:#{error.class} error:#{error}"
         end

data/lib/datadog/core/utils/at_fork_monkey_patch.rb

@@ -51,13 +51,13 @@ module Datadog
           def fork
             # If a block is provided, it must be wrapped to trigger callbacks.
             child_block = if block_given?
-                            proc do
-                              AtForkMonkeyPatch.run_at_fork_blocks(:child)
+              proc do
+                AtForkMonkeyPatch.run_at_fork_blocks(:child)
 
-                              # Invoke original block
-                              yield
-                            end
-                          end
+                # Invoke original block
+                yield
+              end
+            end
 
             # Start fork
             # If a block is provided, use the wrapped version.

data/lib/datadog/core/utils/duration.rb

@@ -7,42 +7,42 @@ module Datadog
       module Duration
         def self.call(value, base: :s)
           cast = if value.include?('.')
-                   method(:Float)
-                 else
-                   method(:Integer)
-                 end
+            method(:Float)
+          else
+            method(:Integer)
+          end
 
           scale = case base
-                  when :s
-                    1_000_000_000
-                  when :ms
-                    1_000_000
-                  when :us
-                    1000
-                  when :ns
-                    1
-                  else
-                    raise ArgumentError, "invalid base: #{base.inspect}"
-                  end
+          when :s
+            1_000_000_000
+          when :ms
+            1_000_000
+          when :us
+            1000
+          when :ns
+            1
+          else
+            raise ArgumentError, "invalid base: #{base.inspect}"
+          end
 
           result = case value
-                   when /^(\d+(?:\.\d+)?)h$/
-                     cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 * 60 / scale
-                   when /^(\d+(?:\.\d+)?)m$/
-                     cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 / scale
-                   when /^(\d+(?:\.\d+)?)s$/
-                     cast.call(Regexp.last_match(1)) * 1_000_000_000 / scale
-                   when /^(\d+(?:\.\d+)?)ms$/
-                     cast.call(Regexp.last_match(1)) * 1_000_000 / scale
-                   when /^(\d+(?:\.\d+)?)us$/
-                     cast.call(Regexp.last_match(1)) * 1_000 / scale
-                   when /^(\d+(?:\.\d+)?)ns$/
-                     cast.call(Regexp.last_match(1)) / scale
-                   when /^(\d+(?:\.\d+)?)$/
-                     cast.call(Regexp.last_match(1))
-                   else
-                     raise ArgumentError, "invalid duration: #{value.inspect}"
-                   end
+          when /^(\d+(?:\.\d+)?)h$/
+            cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 * 60 / scale
+          when /^(\d+(?:\.\d+)?)m$/
+            cast.call(Regexp.last_match(1)) * 1_000_000_000 * 60 / scale
+          when /^(\d+(?:\.\d+)?)s$/
+            cast.call(Regexp.last_match(1)) * 1_000_000_000 / scale
+          when /^(\d+(?:\.\d+)?)ms$/
+            cast.call(Regexp.last_match(1)) * 1_000_000 / scale
+          when /^(\d+(?:\.\d+)?)us$/
+            cast.call(Regexp.last_match(1)) * 1_000 / scale
+          when /^(\d+(?:\.\d+)?)ns$/
+            cast.call(Regexp.last_match(1)) / scale
+          when /^(\d+(?:\.\d+)?)$/
+            cast.call(Regexp.last_match(1))
+          else
+            raise ArgumentError, "invalid duration: #{value.inspect}"
+          end
           # @type var result: Numeric
           result.round
         end

data/lib/datadog/core/utils/forking.rb

@@ -47,12 +47,12 @@ module Datadog
           # This wrapper prevents this by initializing the fork PID when the object is created.
           if RUBY_VERSION >= '3'
             def initialize(*args, **kwargs, &block)
-              super(*args, **kwargs, &block)
+              super
               update_fork_pid!
             end
           else
             def initialize(*args, &block)
-              super(*args, &block)
+              super
               update_fork_pid!
             end
           end