datadog 2.8.0 → 2.39.0

data/lib/datadog/core/telemetry/worker.rb

@@ -1,15 +1,18 @@
 # frozen_string_literal: true
 
-require_relative 'event'
+require_relative "event"
 
-require_relative '../utils/only_once_successful'
-require_relative '../workers/polling'
-require_relative '../workers/queue'
+require_relative "../utils/only_once_successful"
+require_relative "../workers/polling"
+require_relative "../workers/queue"
 
 module Datadog
   module Core
     module Telemetry
-      # Accumulates events and sends them to the API at a regular interval, including heartbeat event.
+      # Accumulates events and sends them to the API at a regular interval,
+      # including heartbeat event.
+      #
+      # @api private
       class Worker
         include Core::Workers::Queue
         include Core::Workers::Polling
@@ -17,14 +20,16 @@ module Datadog
         DEFAULT_BUFFER_MAX_SIZE = 1000
         APP_STARTED_EVENT_RETRIES = 10
 
-        TELEMETRY_STARTED_ONCE = Utils::OnlyOnceSuccessful.new(APP_STARTED_EVENT_RETRIES)
-
         def initialize(
           heartbeat_interval_seconds:,
           metrics_aggregation_interval_seconds:,
           emitter:,
           metrics_manager:,
           dependency_collection:,
+          logger:,
+          settings:,
+          agent_settings:,
+          extended_heartbeat_interval_seconds:,
           enabled: true,
           shutdown_timeout: Workers::Polling::DEFAULT_SHUTDOWN_TIMEOUT,
           buffer_size: DEFAULT_BUFFER_MAX_SIZE
@@ -32,26 +37,57 @@ module Datadog
           @emitter = emitter
           @metrics_manager = metrics_manager
           @dependency_collection = dependency_collection
+          @logger = logger
+          @settings = settings
+          @agent_settings = agent_settings
 
           @ticks_per_heartbeat = (heartbeat_interval_seconds / metrics_aggregation_interval_seconds).to_i
+          @ticks_per_extended_heartbeat = (extended_heartbeat_interval_seconds / metrics_aggregation_interval_seconds).to_i
           @current_ticks = 0
 
           # Workers::Polling settings
           self.enabled = enabled
           # Workers::IntervalLoop settings
           self.loop_base_interval = metrics_aggregation_interval_seconds
-          self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_STOP
+          # We actually restart the worker after fork, but this is done
+          # via the AtForkMonkeyPatch rather than the worker fork policy
+          # because we also need to reset state outside of the worker
+          # (e.g. the metrics).
+          self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_RESTART
 
           @shutdown_timeout = shutdown_timeout
           @buffer_size = buffer_size
 
+          initialize_state
+        end
+
+        # To make the method calls clear, the initialization code is in this
+        # method called +initialize_state+ which is called from +after_fork+.
+        # This way users of this class (e.g. telemetry Component) do not
+        # need to invoke +initialize_state+ directly, which can be confusing.
+        private def initialize_state
           self.buffer = buffer_klass.new(@buffer_size)
+
+          @initial_event_once = Utils::OnlyOnceSuccessful.new(APP_STARTED_EVENT_RETRIES)
+          @extended_heartbeat_ticks = 0
         end
 
-        def start
-          return if !enabled? || forked?
+        attr_reader :logger
+        attr_reader :initial_event_once
+        attr_reader :initial_event
+        attr_reader :emitter
+
+        # Returns true if worker thread is successfully started,
+        # false if worker thread was not started but telemetry is enabled,
+        # nil if telemetry is disabled.
+        def start(initial_event)
+          return unless enabled?
+
+          @initial_event = initial_event
 
           # starts async worker
+          # perform should return true if thread was actually started,
+          # false otherwise
           perform
         end
 
@@ -61,32 +97,74 @@ module Datadog
           super
         end
 
+        # Returns true if event was enqueued, nil if not.
+        # While returning false may seem more reasonable, the only reason
+        # for not enqueueing event (presently) is that telemetry is disabled
+        # altogether, and in this case other methods return nil.
         def enqueue(event)
-          return if !enabled? || forked?
+          return unless enabled?
 
           buffer.push(event)
+          true
+        end
+
+        def sent_initial_event?
+          initial_event_once.success?
         end
 
-        def sent_started_event?
-          TELEMETRY_STARTED_ONCE.success?
+        def failed_initial_event?
+          initial_event_once.failed?
         end
 
-        def failed_to_start?
-          TELEMETRY_STARTED_ONCE.failed?
+        def need_initial_event?
+          !sent_initial_event? && !failed_initial_event?
+        end
+
+        # Wait for the worker to send out all events that have already
+        # been queued, up to 15 seconds. Returns whether all events have
+        # been flushed.
+        #
+        # @api private
+        def flush(timeout: nil)
+          # Increase default timeout to 15 seconds - see the comment in
+          # +idle?+ for more details.
+          super(timeout: timeout || 15)
         end
 
         private
 
         def perform(*events)
-          return if !enabled? || forked?
+          return unless enabled?
 
-          started! unless sent_started_event?
+          if need_initial_event?
+            started!
+            unless sent_initial_event?
+              # We still haven't succeeded in sending the started event,
+              # which will make flush_events do nothing - but the events
+              # given to us as the parameter have already been removed
+              # from the queue.
+              # Put the events back to the front of the queue to not
+              # lose them.
+              buffer.unshift(*events)
+              return
+            end
+          end
 
           metric_events = @metrics_manager.flush!
           events = [] if events.nil?
-          flush_events(events + metric_events)
+          events += metric_events
+          if events.any?
+            flush_events(events)
+          end
 
           @current_ticks += 1
+          @extended_heartbeat_ticks += 1
+
+          if @extended_heartbeat_ticks >= @ticks_per_extended_heartbeat
+            @extended_heartbeat_ticks = 0
+            extended_heartbeat!
+          end
+
           return if @current_ticks < @ticks_per_heartbeat
 
           @current_ticks = 0
@@ -94,42 +172,52 @@ module Datadog
         end
 
         def flush_events(events)
-          return if events.empty?
-          return if !enabled? || !sent_started_event?
+          events = deduplicate_logs(events)
 
-          Datadog.logger.debug { "Sending #{events&.count} telemetry events" }
+          logger.debug { "Sending #{events&.count} telemetry events" }
           send_event(Event::MessageBatch.new(events))
         end
 
         def heartbeat!
-          return if !enabled? || !sent_started_event?
+          return if !enabled? || !sent_initial_event?
 
           send_event(Event::AppHeartbeat.new)
         end
 
+        def extended_heartbeat!
+          return if !enabled? || !sent_initial_event?
+
+          send_event(Event::AppExtendedHeartbeat.new(settings: @settings, agent_settings: @agent_settings))
+        end
+
         def started!
           return unless enabled?
 
-          if failed_to_start?
-            Datadog.logger.debug('Telemetry app-started event exhausted retries, disabling telemetry worker')
-            disable!
-            return
-          end
-
-          TELEMETRY_STARTED_ONCE.run do
-            res = send_event(Event::AppStarted.new)
+          initial_event_once.run do
+            res = send_event(initial_event)
 
             if res.ok?
-              Datadog.logger.debug('Telemetry app-started event is successfully sent')
+              logger.debug { "Telemetry initial event (#{initial_event.type}) is successfully sent" }
 
-              send_event(Event::AppDependenciesLoaded.new) if @dependency_collection
+              # TODO Dependencies loaded event should probably check for new
+              # dependencies and send the new ones.
+              # System tests demand only one instance of this event per
+              # dependency.
+              if @dependency_collection && initial_event.app_started?
+                send_event(Event::AppDependenciesLoaded.new)
+              end
 
               true
             else
-              Datadog.logger.debug('Error sending telemetry app-started event, retry after heartbeat interval...')
+              logger.debug("Error sending telemetry initial event (#{initial_event.type}), retry after heartbeat interval...")
               false
             end
           end
+
+          if failed_initial_event?
+            logger.debug { "Telemetry initial event (#{initial_event.type}) exhausted retries, disabling telemetry worker" }
+            disable!
+          end
         end
 
         def send_event(event)
@@ -140,6 +228,8 @@ module Datadog
           res
         end
 
+        # This method overrides Queue's dequeue method and does LIFO instead
+        # of FIFO that Queue implements. Why?
         def dequeue
           buffer.pop
         end
@@ -149,7 +239,7 @@ module Datadog
         end
 
         def buffer_klass
-          if Core::Environment::Ext::RUBY_ENGINE == 'ruby'
+          if Core::Environment::Ext::RUBY_ENGINE == "ruby"
             Core::Buffer::CRuby
           else
             Core::Buffer::ThreadSafe
@@ -164,9 +254,98 @@ module Datadog
         def disable_on_not_found!(response)
           return unless response.not_found?
 
-          Datadog.logger.debug('Agent does not support telemetry; disabling future telemetry events.')
+          logger.debug("Agent does not support telemetry; disabling future telemetry events.")
           disable!
         end
+
+        # Call this method in a forked child to reset the state of this worker.
+        #
+        # Discard any accumulated events since they will be sent by
+        # the parent.
+        # Discard any accumulated metrics.
+        # Restart the worker thread, if it was running in the parent process.
+        #
+        # This method cannot be called +after_fork+ because workers define
+        # and call +after_fork+ which is supposed to do different things.
+        def after_fork_monkey_patched
+          # If telemetry is disabled, we still reset the state to avoid
+          # having wrong state. It is possible that in the future telemetry
+          # will be re-enabled after errors.
+          initialize_state
+          # In the child process, we get a new runtime_id.
+          # As such we need to send AppStarted event.
+          # In the parent process, the event may have been the
+          # SynthAppClientConfigurationChange instead of AppStarted,
+          # and in that case we need to convert it to the "regular"
+          # AppStarted event.
+          if defined?(@initial_event) && @initial_event.is_a?(Event::SynthAppClientConfigurationChange)
+            # It would be great to just replace the initial event in
+            # +initialize_state+ method. Unfortunately this event requires
+            # the entire component tree to build its payload, which we
+            # 1) don't currently have in telemetry and
+            # 2) don't want to keep a permanent reference to in any case.
+            # Therefore we have this +reset!+ method that changes the
+            # event type while keeping the payload.
+            @initial_event.reset! # steep:ignore NoMethod
+          end
+
+          if enabled? && !worker.nil?
+            # Start the background thread if it was started in the parent
+            # process (which requires telemetry to be enabled).
+            # This should be done after all of the state resets.
+            perform
+          end
+        end
+
+        # Deduplicate logs by counting the number of repeated occurrences of the same log
+        # entry and replacing them with a single entry with the calculated `count` value.
+        # Non-log events are unchanged.
+        def deduplicate_logs(events)
+          return events if events.empty?
+
+          all_logs = []
+          other_events = events.reject do |event|
+            if event.is_a?(Event::Log)
+              all_logs << event
+              true
+            else
+              false
+            end
+          end
+
+          return events if all_logs.empty?
+
+          uniq_logs = all_logs.group_by(&:itself).map do |_, logs|
+            log = logs.first
+            if logs.size > 1
+              # New log event with a count of repeated occurrences
+              Event::Log.new(message: log.message, level: log.level, stack_trace: log.stack_trace, count: logs.size)
+            else
+              log
+            end
+          end
+
+          other_events + uniq_logs
+        end
+
+        def idle?
+          # The AppStarted event is triggered by the worker itself,
+          # from the worker thread. As such the main thread has no way
+          # to delay itself until that event is queued and we need some
+          # way to wait until that event is sent out to assert on it in
+          # the test suite. Check the run once flag which *should*
+          # indicate the event has been queued (at which point our queue
+          # depth check should wait until it's sent).
+          # This is still a hack because the flag can be overridden
+          # either way with or without the event being sent out.
+          # Note that if the AppStarted sending fails, this check
+          # will return false and flushing will be blocked until the
+          # 15 second timeout.
+          # Note that the first wait interval between telemetry event
+          # sending is 10 seconds, the timeout needs to be strictly
+          # greater than that.
+          super && sent_initial_event?
+        end
       end
     end
   end

data/lib/datadog/core/transport/ext.rb

@@ -1,32 +1,35 @@
 # frozen_string_literal: true
 
-require_relative '../configuration/ext'
+require_relative "../configuration/ext"
 module Datadog
   module Core
     module Transport
       # @public_api
       module Ext
         module HTTP
-          HEADER_CONTAINER_ID = 'Datadog-Container-ID'
-          HEADER_DD_API_KEY = 'DD-API-KEY'
+          HEADER_CONTAINER_ID = "Datadog-Container-ID"
+          HEADER_CONTAINER_TAGS_HASH = "Datadog-Container-Tags-Hash"
+          HEADER_ENTITY_ID = "Datadog-Entity-ID"
+          HEADER_EXTERNAL_ENV = "Datadog-External-Env"
+          HEADER_DD_API_KEY = "DD-API-KEY"
           # Tells agent that `_dd.top_level` metrics have been set by the tracer.
           # The agent will not calculate top-level spans but instead trust the tracer tagging.
           #
           # This prevents partially flushed traces being mistakenly marked as top-level.
           #
           # Setting this header to any non-empty value enables this feature.
-          HEADER_CLIENT_COMPUTED_TOP_LEVEL = 'Datadog-Client-Computed-Top-Level'
-          HEADER_CLIENT_COMPUTED_STATS = 'Datadog-Client-Computed-Stats'
-          HEADER_META_LANG = 'Datadog-Meta-Lang'
-          HEADER_META_LANG_VERSION = 'Datadog-Meta-Lang-Version'
-          HEADER_META_LANG_INTERPRETER = 'Datadog-Meta-Lang-Interpreter'
+          HEADER_CLIENT_COMPUTED_TOP_LEVEL = "Datadog-Client-Computed-Top-Level"
+          HEADER_CLIENT_COMPUTED_STATS = "Datadog-Client-Computed-Stats"
+          HEADER_META_LANG = "Datadog-Meta-Lang"
+          HEADER_META_LANG_VERSION = "Datadog-Meta-Lang-Version"
+          HEADER_META_LANG_INTERPRETER = "Datadog-Meta-Lang-Interpreter"
           # Use for distinguishing between CRuby, JRuby, and TruffleRuby.
-          HEADER_META_LANG_INTERPRETER_VENDOR = 'Datadog-Meta-Lang-Interpreter-Vendor'
-          HEADER_META_TRACER_VERSION = 'Datadog-Meta-Tracer-Version'
+          HEADER_META_LANG_INTERPRETER_VENDOR = "Datadog-Meta-Lang-Interpreter-Vendor"
+          HEADER_META_TRACER_VERSION = "Datadog-Meta-Tracer-Version"
 
           # Header that prevents the Net::HTTP integration from tracing internal trace requests.
           # Set it to any value to skip tracing.
-          HEADER_DD_INTERNAL_UNTRACED_REQUEST = 'DD-Internal-Untraced-Request'
+          HEADER_DD_INTERNAL_UNTRACED_REQUEST = "DD-Internal-Untraced-Request"
         end
 
         module Test

data/lib/datadog/core/transport/http/adapters/net.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require_relative '../../../../core/transport/response'
-require_relative '../../../../core/vendor/multipart-post/net/http/post/multipart'
+require_relative "../../../../core/transport/response"
+require_relative "../../../../core/vendor/multipart-post/net/http/post/multipart"
 
 module Datadog
   module Core
@@ -48,7 +48,7 @@ module Datadog
             end
 
             def get(env)
-              get = ::Net::HTTP::Get.new(env.path, env.headers)
+              get = ::Net::HTTP::Get.new(net_http_path_from_env(env), env.headers)
 
               # Connect and send the request
               http_response = open do |http|
@@ -63,7 +63,7 @@ module Datadog
               post = nil
 
               if env.form.nil? || env.form.empty?
-                post = ::Net::HTTP::Post.new(env.path, env.headers)
+                post = ::Net::HTTP::Post.new(net_http_path_from_env(env), env.headers)
                 post.body = env.body
               else
                 post = ::Datadog::Core::Vendor::Net::HTTP::Post::Multipart.new(
@@ -86,6 +86,21 @@ module Datadog
               "http://#{hostname}:#{port}?timeout=#{timeout}"
             end
 
+            def net_http_path_from_env(env)
+              path = env.path
+              case query = env.query
+              when Hash
+                path = path + "?" + URI.encode_www_form(query)
+              when String
+                path = path + "?" + query
+              when nil
+                # Nothing
+              else
+                raise ArgumentError, "Invalid type for query: #{query}"
+              end
+              path
+            end
+
             # Raised when called with an unknown HTTP method
             class UnknownHTTPMethod < StandardError
               attr_reader :verb

data/lib/datadog/core/transport/http/adapters/test.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative '../../../../core/transport/response'
+require_relative "../../../../core/transport/response"
 
 module Datadog
   module Core
@@ -38,7 +38,8 @@ module Datadog
               @status = status
             end
 
-            def url; end
+            def url
+            end
 
             # Response for test adapter
             class Response

data/lib/datadog/core/transport/http/adapters/unix_socket.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'net/http'
-require_relative '../../../transport/ext'
-require_relative 'net'
+require "net/http"
+require_relative "../../../transport/ext"
+require_relative "net"
 
 module Datadog
   module Core
@@ -49,7 +49,7 @@ module Datadog
 
             # Re-implements Net:HTTP with underlying Unix socket
             class HTTP < ::Net::HTTP
-              DEFAULT_TIMEOUT = 1
+              DEFAULT_TIMEOUT = 30
 
               attr_reader \
                 :filepath, # DEV(1.0): Rename to `uds_path`
@@ -58,7 +58,7 @@ module Datadog
               alias_method :uds_path, :filepath
 
               def initialize(uds_path, options = {})
-                super('localhost', 80)
+                super("localhost", 80)
                 @filepath = uds_path
                 @read_timeout = options.fetch(:read_timeout, DEFAULT_TIMEOUT)
                 @continue_timeout = options.fetch(:continue_timeout, DEFAULT_TIMEOUT)

data/lib/datadog/core/transport/http/api/endpoint.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'json'
+require "json"
 
 module Datadog
   module Core
@@ -9,13 +9,18 @@ module Datadog
         module API
           # Endpoint
           class Endpoint
-            attr_reader \
-              :verb,
-              :path
+            attr_reader :verb
+            attr_reader :path
 
-            def initialize(verb, path)
+            # TODO Currently only Traces transport specifies an encoder.
+            # Other transports perform encoding "inline" / ad-hoc.
+            # They should probably use this encoder field instead.
+            attr_reader :encoder
+
+            def initialize(verb, path, encoder: nil)
               @verb = verb
               @path = path
+              @encoder = encoder
             end
 
             def call(env)

data/lib/datadog/{tracing → core}/transport/http/api/instance.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Datadog
-  module Tracing
+  module Core
     module Transport
       module HTTP
         module API
@@ -10,16 +10,16 @@ module Datadog
             attr_reader \
               :adapter,
               :headers,
-              :spec
+              :endpoint
 
-            def initialize(spec, adapter, options = {})
-              @spec = spec
+            def initialize(endpoint, adapter, options = {})
+              @endpoint = endpoint
               @adapter = adapter
               @headers = options.fetch(:headers, {})
             end
 
             def encoder
-              spec.encoder
+              endpoint.encoder
             end
 
             def call(env)

data/lib/datadog/core/transport/http/api/map.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'fallbacks'
+require_relative "fallbacks"
 
 module Datadog
   module Core