datadog 2.11.0 → 2.12.1

data/lib/datadog/core/remote/worker.rb

@@ -5,7 +5,7 @@ module Datadog
     module Remote
       # Worker executes a block every interval on a separate Thread
       class Worker
-        def initialize(interval:, &block)
+        def initialize(interval:, logger:, &block)
           @mutex = Mutex.new
           @thr = nil
 
@@ -14,18 +14,21 @@ module Datadog
           @stopped = false
 
           @interval = interval
+          @logger = logger
           raise ArgumentError, 'can not initialize a worker without a block' unless block
 
           @block = block
         end
 
+        attr_reader :logger
+
         def start
-          Datadog.logger.debug { 'remote worker starting' }
+          logger.debug { 'remote worker starting' }
 
           acquire_lock
 
           if @stopped
-            Datadog.logger.debug('remote worker: refusing to restart after previous stop')
+            logger.debug('remote worker: refusing to restart after previous stop')
             return
           end
 
@@ -41,13 +44,13 @@ module Datadog
           @started = true
           @starting = false
 
-          Datadog.logger.debug { 'remote worker started' }
+          logger.debug { 'remote worker started' }
         ensure
           release_lock
         end
 
         def stop
-          Datadog.logger.debug { 'remote worker stopping' }
+          logger.debug { 'remote worker stopping' }
 
           acquire_lock
 
@@ -62,7 +65,7 @@ module Datadog
           @thr = nil
           @stopped = true
 
-          Datadog.logger.debug { 'remote worker stopped' }
+          logger.debug { 'remote worker stopped' }
         ensure
           release_lock
         end
@@ -92,7 +95,7 @@ module Datadog
         end
 
         def call
-          Datadog.logger.debug { 'remote worker perform' }
+          logger.debug { 'remote worker perform' }
 
           @block.call
         end

data/lib/datadog/core/telemetry/component.rb

@@ -16,7 +16,7 @@ module Datadog
       # Telemetry entrypoint, coordinates sending telemetry events at various points in app lifecycle.
       # Note: Telemetry does not spawn its worker thread in fork processes, thus no telemetry is sent in forked processes.
       class Component
-        attr_reader :enabled
+        attr_reader :enabled, :logger
 
         include Core::Utils::Forking
         include Telemetry::Logging
@@ -52,6 +52,7 @@ module Datadog
             heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
             metrics_aggregation_interval_seconds: settings.telemetry.metrics_aggregation_interval_seconds,
             dependency_collection: settings.telemetry.dependency_collection,
+            logger: logger,
             shutdown_timeout_seconds: settings.telemetry.shutdown_timeout_seconds,
             log_collection_enabled: settings.telemetry.log_collection_enabled
           )
@@ -66,6 +67,7 @@ module Datadog
           heartbeat_interval_seconds:,
           metrics_aggregation_interval_seconds:,
           dependency_collection:,
+          logger:,
           http_transport:,
           shutdown_timeout_seconds:,
           enabled: true,
@@ -74,6 +76,7 @@ module Datadog
         )
           @enabled = enabled
           @log_collection_enabled = log_collection_enabled
+          @logger = logger
 
           @metrics_manager = MetricsManager.new(
             enabled: enabled && metrics_enabled,
@@ -87,6 +90,7 @@ module Datadog
             emitter: Emitter.new(http_transport: http_transport),
             metrics_manager: @metrics_manager,
             dependency_collection: dependency_collection,
+            logger: logger,
             shutdown_timeout: shutdown_timeout_seconds
           )
 

data/lib/datadog/core/telemetry/worker.rb

@@ -25,6 +25,7 @@ module Datadog
           emitter:,
           metrics_manager:,
           dependency_collection:,
+          logger:,
           enabled: true,
           shutdown_timeout: Workers::Polling::DEFAULT_SHUTDOWN_TIMEOUT,
           buffer_size: DEFAULT_BUFFER_MAX_SIZE
@@ -32,6 +33,7 @@ module Datadog
           @emitter = emitter
           @metrics_manager = metrics_manager
           @dependency_collection = dependency_collection
+          @logger = logger
 
           @ticks_per_heartbeat = (heartbeat_interval_seconds / metrics_aggregation_interval_seconds).to_i
           @current_ticks = 0
@@ -48,6 +50,8 @@ module Datadog
           self.buffer = buffer_klass.new(@buffer_size)
         end
 
+        attr_reader :logger
+
         def start
           return if !enabled? || forked?
 
@@ -99,7 +103,7 @@ module Datadog
 
           events = deduplicate_logs(events)
 
-          Datadog.logger.debug { "Sending #{events&.count} telemetry events" }
+          logger.debug { "Sending #{events&.count} telemetry events" }
           send_event(Event::MessageBatch.new(events))
         end
 
@@ -113,7 +117,7 @@ module Datadog
           return unless enabled?
 
           if failed_to_start?
-            Datadog.logger.debug('Telemetry app-started event exhausted retries, disabling telemetry worker')
+            logger.debug('Telemetry app-started event exhausted retries, disabling telemetry worker')
             disable!
             return
           end
@@ -122,13 +126,13 @@ module Datadog
             res = send_event(Event::AppStarted.new)
 
             if res.ok?
-              Datadog.logger.debug('Telemetry app-started event is successfully sent')
+              logger.debug('Telemetry app-started event is successfully sent')
 
               send_event(Event::AppDependenciesLoaded.new) if @dependency_collection
 
               true
             else
-              Datadog.logger.debug('Error sending telemetry app-started event, retry after heartbeat interval...')
+              logger.debug('Error sending telemetry app-started event, retry after heartbeat interval...')
               false
             end
           end
@@ -166,7 +170,7 @@ module Datadog
         def disable_on_not_found!(response)
           return unless response.not_found?
 
-          Datadog.logger.debug('Agent does not support telemetry; disabling future telemetry events.')
+          logger.debug('Agent does not support telemetry; disabling future telemetry events.')
           disable!
         end
 

data/lib/datadog/core/transport/http.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative 'http/builder'
+require_relative 'http/adapters/net'
+require_relative 'http/adapters/unix_socket'
+require_relative 'http/adapters/test'
+
+module Datadog
+  module Core
+    module Transport
+      # HTTP transport
+      module HTTP
+        # Add adapters to registry
+        Builder::REGISTRY.set(
+          Transport::HTTP::Adapters::Net,
+          Core::Configuration::Ext::Agent::HTTP::ADAPTER
+        )
+        Builder::REGISTRY.set(
+          Transport::HTTP::Adapters::Test,
+          Transport::Ext::Test::ADAPTER
+        )
+        Builder::REGISTRY.set(
+          Transport::HTTP::Adapters::UnixSocket,
+          Transport::Ext::UnixSocket::ADAPTER
+        )
+
+        module_function
+
+        # Helper function that delegates to Builder.new
+        # but is under HTTP namespace so that client code requires this file
+        # to get the adapters configured, and not the builder directly.
+        def build(api_instance_class:, &block)
+          Builder.new(api_instance_class: api_instance_class, &block)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/workers/runtime_metrics.rb

@@ -21,7 +21,7 @@ module Datadog
           :metrics
 
         def initialize(options = {})
-          @metrics = options.fetch(:metrics) { Core::Runtime::Metrics.new }
+          @metrics = options.fetch(:metrics) { Core::Runtime::Metrics.new(logger: options[:logger]) }
 
           # Workers::Async::Thread settings
           self.fork_policy = options.fetch(:fork_policy, Workers::Async::Thread::FORK_POLICY_STOP)

data/lib/datadog/di/component.rb

@@ -81,8 +81,7 @@ module Datadog
         @redactor = Redactor.new(settings)
         @serializer = Serializer.new(settings, redactor, telemetry: telemetry)
         @instrumenter = Instrumenter.new(settings, serializer, logger, code_tracker: code_tracker, telemetry: telemetry)
-        @transport = Transport.new(agent_settings)
-        @probe_notifier_worker = ProbeNotifierWorker.new(settings, transport, logger, telemetry: telemetry)
+        @probe_notifier_worker = ProbeNotifierWorker.new(settings, logger, agent_settings: agent_settings, telemetry: telemetry)
         @probe_notification_builder = ProbeNotificationBuilder.new(settings, serializer)
         @probe_manager = ProbeManager.new(settings, instrumenter, probe_notification_builder, probe_notifier_worker, logger, telemetry: telemetry)
         probe_notifier_worker.start
@@ -94,7 +93,6 @@ module Datadog
       attr_reader :telemetry
       attr_reader :code_tracker
       attr_reader :instrumenter
-      attr_reader :transport
       attr_reader :probe_notifier_worker
       attr_reader :probe_notification_builder
       attr_reader :probe_manager

data/lib/datadog/di/probe_notifier_worker.rb

@@ -23,12 +23,12 @@ module Datadog
     #
     # @api private
     class ProbeNotifierWorker
-      def initialize(settings, transport, logger, telemetry: nil)
+      def initialize(settings, logger, agent_settings:, telemetry: nil)
         @settings = settings
         @telemetry = telemetry
         @status_queue = []
         @snapshot_queue = []
-        @transport = transport
+        @agent_settings = agent_settings
         @logger = logger
         @lock = Mutex.new
         @wake = Core::Semaphore.new
@@ -43,6 +43,7 @@ module Datadog
       attr_reader :settings
       attr_reader :logger
       attr_reader :telemetry
+      attr_reader :agent_settings
 
       def start
         return if @thread && @pid == Process.pid
@@ -154,7 +155,6 @@ module Datadog
 
       private
 
-      attr_reader :transport
       attr_reader :wake
       attr_reader :thread
 
@@ -170,6 +170,22 @@ module Datadog
 
       attr_reader :last_sent
 
+      def status_transport
+        @status_transport ||= DI::Transport::HTTP.diagnostics(agent_settings: agent_settings)
+      end
+
+      def do_send_status(batch)
+        status_transport.send_diagnostics(batch)
+      end
+
+      def snapshot_transport
+        @snapshot_transport ||= DI::Transport::HTTP.input(agent_settings: agent_settings)
+      end
+
+      def do_send_snapshot(batch)
+        snapshot_transport.send_input(batch)
+      end
+
       [
         [:status, 'probe status'],
         [:snapshot, 'snapshot'],
@@ -245,7 +261,7 @@ module Datadog
           if batch.any? # steep:ignore
             begin
               logger.trace { "di: sending #{batch.length} #{event_type} event(s) to agent" } # steep:ignore
-              transport.public_send("send_#{event_type}", batch)
+              send("do_send_#{event_type}", batch)
               time = Core::Utils::Time.get_time
               @lock.synchronize do
                 @last_sent = time

data/lib/datadog/di/transport/diagnostics.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative '../../core/transport/parcel'
+require_relative 'http/client'
+
+module Datadog
+  module DI
+    module Transport
+      module Diagnostics
+        class EncodedParcel
+          include Datadog::Core::Transport::Parcel
+        end
+
+        class Request < Datadog::Core::Transport::Request
+        end
+
+        class Transport
+          attr_reader :client, :apis, :default_api, :current_api_id
+
+          def initialize(apis, default_api)
+            @apis = apis
+
+            @client = HTTP::Client.new(current_api)
+          end
+
+          def current_api
+            @apis[HTTP::API::DIAGNOSTICS]
+          end
+
+          def send_diagnostics(payload)
+            json = JSON.dump(payload)
+            parcel = EncodedParcel.new(json)
+            request = Request.new(parcel)
+
+            response = @client.send_diagnostics_payload(request)
+            unless response.ok?
+              # TODO Datadog::Core::Transport::InternalErrorResponse
+              # does not have +code+ method, what is the actual API of
+              # these response objects?
+              raise Error::AgentCommunicationError, "send_diagnostics failed: #{begin
+                response.code
+              rescue
+                "???"
+              end}: #{response.payload}"
+            end
+          rescue Error::AgentCommunicationError
+            raise
+          # Datadog::Core::Transport does not perform any exception mapping,
+          # therefore we could have any exception here from failure to parse
+          # agent URI for example.
+          # If we ever implement retries for network errors, we should distinguish
+          # actual network errors from non-network errors that are raised by
+          # transport code.
+          rescue => exc
+            raise Error::AgentCommunicationError, "send_diagnostics failed: #{exc.class}: #{exc}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http/api.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/encoding'
+require_relative '../../../core/transport/http/api/map'
+require_relative '../../../core/transport/http/api/instance'
+require_relative '../../../core/transport/http/api/spec'
+require_relative 'diagnostics'
+require_relative 'input'
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        # Namespace for API components
+        module API
+          # Default API versions
+          DIAGNOSTICS = 'diagnostics'
+          INPUT = 'input'
+
+          module_function
+
+          def defaults
+            Datadog::Core::Transport::HTTP::API::Map[
+              DIAGNOSTICS => Spec.new do |s|
+                s.diagnostics = Diagnostics::API::Endpoint.new(
+                  '/debugger/v1/diagnostics',
+                  Core::Encoding::JSONEncoder,
+                )
+              end,
+              INPUT => Spec.new do |s|
+                s.input = Input::API::Endpoint.new(
+                  '/debugger/v1/input',
+                  Core::Encoding::JSONEncoder,
+                )
+              end,
+            ]
+          end
+
+          class Instance < Core::Transport::HTTP::API::Instance
+            include Diagnostics::API::Instance
+            include Input::API::Instance
+          end
+
+          class Spec < Core::Transport::HTTP::API::Spec
+            include Diagnostics::API::Spec
+            include Input::API::Spec
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http/client.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/transport/http/env'
+require_relative '../../../core/transport/http/response'
+
+# TODO: Decouple transport/http/client
+#
+# The standard one does `include Transport::HTTP::Statistics` and performs
+# stats updates, which may or may not be desirable in general.
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        # Routes, encodes, and sends DI data to the trace agent via HTTP.
+        class Client
+          attr_reader :api
+
+          def initialize(api)
+            @api = api
+          end
+
+          def send_request(request, &block)
+            # Build request into env
+            env = build_env(request)
+
+            # Get responses from API
+            yield(api, env)
+          rescue => e
+            message =
+              "Internal error during #{self.class.name} request. Cause: #{e.class.name} #{e.message} " \
+                "Location: #{Array(e.backtrace).first}"
+
+            Datadog.logger.debug(message)
+
+            Datadog::Core::Transport::InternalErrorResponse.new(e)
+          end
+
+          def build_env(request)
+            Datadog::Core::Transport::HTTP::Env.new(request)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http/diagnostics.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require_relative 'client'
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        module Diagnostics
+          module Client
+            def send_diagnostics_payload(request)
+              send_request(request) do |api, env|
+                api.send_diagnostics(env)
+              end
+            end
+          end
+
+          module API
+            module Instance
+              def send_diagnostics(env)
+                raise DiagnosticsNotSupportedError, spec unless spec.is_a?(Diagnostics::API::Spec)
+
+                spec.send_diagnostics(env) do |request_env|
+                  call(request_env)
+                end
+              end
+
+              class DiagnosticsNotSupportedError < StandardError
+                attr_reader :spec
+
+                def initialize(spec)
+                  super
+
+                  @spec = spec
+                end
+
+                def message
+                  'Diagnostics not supported for this API!'
+                end
+              end
+            end
+
+            module Spec
+              attr_accessor :diagnostics
+
+              def send_diagnostics(env, &block)
+                raise NoDiagnosticsEndpointDefinedError, self if diagnostics.nil?
+
+                diagnostics.call(env, &block)
+              end
+
+              class NoDiagnosticsEndpointDefinedError < StandardError
+                attr_reader :spec
+
+                def initialize(spec)
+                  super
+
+                  @spec = spec
+                end
+
+                def message
+                  'No diagnostics endpoint is defined for API specification!'
+                end
+              end
+            end
+
+            # Endpoint for negotiation
+            class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
+              attr_reader :encoder
+
+              def initialize(path, encoder)
+                super(:post, path)
+                @encoder = encoder
+              end
+
+              def call(env, &block)
+                event_payload = Core::Vendor::Multipart::Post::UploadIO.new(
+                  StringIO.new(env.request.parcel.data), 'application/json', 'event.json'
+                )
+                env.form = {'event' => event_payload}
+
+                super(env, &block)
+              end
+            end
+          end
+        end
+
+        HTTP::Client.include(Diagnostics::Client)
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http/input.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require_relative 'client'
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        module Input
+          module Client
+            def send_input_payload(request)
+              send_request(request) do |api, env|
+                api.send_input(env)
+              end
+            end
+          end
+
+          module API
+            module Instance
+              def send_input(env)
+                raise InputNotSupportedError, spec unless spec.is_a?(Input::API::Spec)
+
+                spec.send_input(env) do |request_env|
+                  call(request_env)
+                end
+              end
+
+              class InputNotSupportedError < StandardError
+                attr_reader :spec
+
+                def initialize(spec)
+                  super
+
+                  @spec = spec
+                end
+
+                def message
+                  'Input not supported for this API!'
+                end
+              end
+            end
+
+            module Spec
+              attr_accessor :input
+
+              def send_input(env, &block)
+                raise NoInputEndpointDefinedError, self if input.nil?
+
+                input.call(env, &block)
+              end
+
+              class NoInputEndpointDefinedError < StandardError
+                attr_reader :spec
+
+                def initialize(spec)
+                  super
+
+                  @spec = spec
+                end
+
+                def message
+                  'No input endpoint is defined for API specification!'
+                end
+              end
+            end
+
+            # Endpoint for negotiation
+            class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
+              HEADER_CONTENT_TYPE = 'Content-Type'
+
+              attr_reader \
+                :encoder
+
+              def initialize(path, encoder)
+                super(:post, path)
+                @encoder = encoder
+              end
+
+              def call(env, &block)
+                # Encode body & type
+                env.headers[HEADER_CONTENT_TYPE] = encoder.content_type
+                env.body = env.request.parcel.data
+
+                super(env, &block)
+              end
+            end
+          end
+        end
+
+        HTTP::Client.include(Input::Client)
+      end
+    end
+  end
+end