datadog 2.15.0 → 2.16.0

data/lib/datadog/core/telemetry/event.rb

@@ -74,10 +74,9 @@ module Datadog
               profiler: {
                 enabled: Datadog::Profiling.enabled?,
               },
-              # DEV: Not implemented yet
-              # dynamic_instrumentation: {
-              #   enabled: true,
-              # }
+              dynamic_instrumentation: {
+                enabled: defined?(Datadog::DI) && Datadog::DI.respond_to?(:enabled?) && Datadog::DI.enabled?,
+              }
             }
 
             if (unsupported_reason = Datadog::Profiling.unsupported_reason)
@@ -91,6 +90,7 @@ module Datadog
           end
 
           TARGET_OPTIONS = %w[
+            dynamic_instrumentation.enabled
             logger.level
             profiling.advanced.code_provenance_enabled
             profiling.advanced.endpoint.collection.enabled
@@ -114,6 +114,9 @@ module Datadog
             seq_id = Event.configuration_sequence.next
 
             list = [
+              conf_value('DD_GIT_REPOSITORY_URL', Core::Environment::Git.git_repository_url, seq_id, 'env_var'),
+              conf_value('DD_GIT_COMMIT_SHA', Core::Environment::Git.git_commit_sha, seq_id, 'env_var'),
+
               conf_value('DD_AGENT_HOST', config.agent.host, seq_id),
               conf_value('DD_AGENT_TRANSPORT', agent_transport(config), seq_id),
               conf_value('DD_TRACE_SAMPLE_RATE', to_value(config.tracing.sampling.default_rate), seq_id),

data/lib/datadog/core/telemetry/http/adapters/net.rb

@@ -1,108 +1,23 @@
 # frozen_string_literal: true
 
-require_relative '../response'
+# datadog-ci-rb versions 1.15.0 and lower require this file and guard
+# the require with a rescue of StandardError. Unfortunately LoadError,
+# which would be raised if the file is missing, is not a subclass of
+# StandardError and thus would not be caught by the rescue.
+# We provide this file with a dummy class in it to avoid exceptions
+# in datadog-ci-rb until version 2.0 is released.
+#
+# Note that datadog-ci-rb patches telemetry transport to be "real" even when
+# webmock is used; this patching won't work with datadog-ci-rb versions
+# 1.15.0 and older and dd-trace-rb 2.16.0 and newer. There will be no
+# errors/exceptions reported but telemetry events will not be sent.
 
 module Datadog
   module Core
     module Telemetry
       module Http
         module Adapters
-          # Class defining methods to make http requests via NET
-          class Net
-            attr_reader \
-              :hostname,
-              :port,
-              :timeout,
-              :ssl
-
-            DEFAULT_TIMEOUT = 2
-
-            def initialize(hostname:, port: nil, timeout: DEFAULT_TIMEOUT, ssl: true)
-              @hostname = hostname
-              @port = port
-              @timeout = timeout
-              @ssl = ssl.nil? || ssl
-            end
-
-            def open(&block)
-              req = ::Net::HTTP.new(@hostname, @port)
-
-              req.use_ssl = @ssl
-              req.open_timeout = req.read_timeout = @timeout
-
-              req.start(&block)
-            end
-
-            def post(env)
-              post = ::Net::HTTP::Post.new(env.path, env.headers)
-              post.body = env.body
-
-              http_response = open do |http|
-                http.request(post)
-              end
-
-              Response.new(http_response)
-            rescue StandardError => e
-              Datadog.logger.debug("Unable to send telemetry event to agent: #{e}")
-              Telemetry::Http::InternalErrorResponse.new(e)
-            end
-
-            # Data structure for an HTTP Response
-            class Response
-              include Datadog::Core::Telemetry::Http::Response
-
-              attr_reader :http_response
-
-              def initialize(http_response)
-                @http_response = http_response
-              end
-
-              def payload
-                return super if http_response.nil?
-
-                http_response.body
-              end
-
-              def code
-                return super if http_response.nil?
-
-                http_response.code.to_i
-              end
-
-              def ok?
-                return super if http_response.nil?
-
-                code.between?(200, 299)
-              end
-
-              def unsupported?
-                return super if http_response.nil?
-
-                code == 415
-              end
-
-              def not_found?
-                return super if http_response.nil?
-
-                code == 404
-              end
-
-              def client_error?
-                return super if http_response.nil?
-
-                code.between?(400, 499)
-              end
-
-              def server_error?
-                return super if http_response.nil?
-
-                code.between?(500, 599)
-              end
-
-              def inspect
-                "#{super}, http_response:#{http_response}"
-              end
-            end
+          class Net # rubocop:disable Lint/EmptyClass
           end
         end
       end

data/lib/datadog/core/telemetry/request.rb

@@ -11,11 +11,11 @@ module Datadog
         class << self
           using Core::Utils::Hash::Refinement
 
-          def build_payload(event, seq_id)
+          def build_payload(event, seq_id, api_version: 'v2', debug: false)
             hash = {
-              api_version: Http::Ext::API_VERSION,
+              api_version: api_version,
               application: application,
-              debug: false,
+              debug: debug,
               host: host,
               payload: event.payload,
               request_type: event.type,

data/lib/datadog/core/telemetry/transport/http/api.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative '../../../encoding'
+require_relative '../../../transport/http/api/map'
+require_relative '../../../transport/http/api/instance'
+require_relative '../../../transport/http/api/spec'
+require_relative 'telemetry'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Transport
+        module HTTP
+          # Namespace for API components
+          module API
+            # Default API versions
+            AGENT_TELEMETRY = 'agent_telemetry'
+            AGENTLESS_TELEMETRY = 'agentless_telemetry'
+
+            module_function
+
+            def defaults
+              Datadog::Core::Transport::HTTP::API::Map[
+                AGENT_TELEMETRY => Telemetry::API::Spec.new do |s|
+                  s.telemetry = Telemetry::API::Endpoint.new(
+                    '/telemetry/proxy/api/v2/apmtelemetry',
+                    Core::Encoding::JSONEncoder,
+                  )
+                end,
+                AGENTLESS_TELEMETRY => Telemetry::API::Spec.new do |s|
+                  s.telemetry = Telemetry::API::Endpoint.new(
+                    '/api/v2/apmtelemetry',
+                    Core::Encoding::JSONEncoder,
+                  )
+                end,
+              ]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/transport/http/client.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative '../../../transport/http/env'
+require_relative '../../../transport/http/response'
+
+# TODO: Decouple transport/http/client
+#
+# The standard one does `include Transport::HTTP::Statistics` and performs
+# stats updates, which may or may not be desirable in general.
+
+module Datadog
+  module Core
+    module Telemetry
+      module Transport
+        module HTTP
+          # Routes, encodes, and sends DI data to the trace agent via HTTP.
+          class Client
+            attr_reader :api, :logger
+
+            def initialize(api, logger:)
+              @api = api
+              @logger = logger
+            end
+
+            def send_request(request, &block)
+              # Build request into env
+              env = build_env(request)
+
+              # Get responses from API
+              yield(api, env)
+            rescue => e
+              message =
+                "Internal error during #{self.class.name} request. Cause: #{e.class.name} #{e.message} " \
+                  "Location: #{Array(e.backtrace).first}"
+
+              logger.debug(message)
+
+              Datadog::Core::Transport::InternalErrorResponse.new(e)
+            end
+
+            def build_env(request)
+              Datadog::Core::Transport::HTTP::Env.new(request)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/transport/http/telemetry.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require_relative '../../../transport/http/api/endpoint'
+require_relative '../../../transport/http/api/instance'
+require_relative '../../../transport/http/api/spec'
+require_relative '../../../transport/request'
+require_relative 'client'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Transport
+        module HTTP
+          module Telemetry
+            module Client
+              def send_telemetry_payload(request)
+                send_request(request) do |api, env| # steep:ignore
+                  api.send_telemetry(env)
+                end
+              end
+            end
+
+            module API
+              class Instance < Core::Transport::HTTP::API::Instance
+                def send_telemetry(env)
+                  raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new('telemetry', self) unless spec.is_a?(Telemetry::API::Spec)
+
+                  spec.send_telemetry(env) do |request_env|
+                    call(request_env)
+                  end
+                end
+              end
+
+              class Spec < Core::Transport::HTTP::API::Spec
+                attr_accessor :telemetry
+
+                def send_telemetry(env, &block)
+                  raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('telemetry', self) if telemetry.nil?
+
+                  telemetry.call(env, &block)
+                end
+              end
+
+              class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
+                HEADER_CONTENT_TYPE = 'Content-Type'
+
+                attr_reader \
+                  :encoder
+
+                def initialize(path, encoder)
+                  super(:post, path)
+                  @encoder = encoder
+                end
+
+                def call(env, &block)
+                  # Encode body & type
+                  env.headers[HEADER_CONTENT_TYPE] = encoder.content_type
+                  env.headers.update(headers(
+                    request_type: env.request.request_type,
+                    api_key: env.request.api_key,
+                  ))
+                  env.body = env.request.parcel.data
+
+                  super
+                end
+
+                def headers(request_type:, api_version: 'v2', api_key:)
+                  {
+                    Core::Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST => '1',
+                    # Provided by encoder
+                    #'Content-Type' => 'application/json',
+                    'DD-Telemetry-API-Version' => api_version,
+                    'DD-Telemetry-Request-Type' => request_type,
+                    'DD-Client-Library-Language' => Core::Environment::Ext::LANG,
+                    'DD-Client-Library-Version' => Core::Environment::Identity.gem_datadog_version_semver2,
+
+                    # Enable debug mode for telemetry
+                    # 'DD-Telemetry-Debug-Enabled' => 'true',
+                  }.tap do |result|
+                    result['DD-API-KEY'] = api_key unless api_key.nil?
+                  end
+                end
+              end
+            end
+          end
+
+          HTTP::Client.include(Telemetry::Client)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/transport/http.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative 'telemetry'
+require_relative 'http/api'
+require_relative '../../transport/http'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Transport
+        # Namespace for HTTP transport components
+        module HTTP
+          module_function
+
+          # Builds a new Transport::HTTP::Client with default settings
+          # Pass a block to override any settings.
+          def agentless_telemetry(
+            agent_settings:,
+            logger:,
+            api_key: nil,
+            api_version: nil,
+            headers: nil
+          )
+            Core::Transport::HTTP.build(api_instance_class: Telemetry::API::Instance,
+              logger: logger,
+              agent_settings: agent_settings,
+              api_version: api_version,
+              headers: headers) do |transport|
+              apis = API.defaults
+
+              transport.api API::AGENTLESS_TELEMETRY, apis[API::AGENTLESS_TELEMETRY]
+
+              # Call block to apply any customization, if provided
+              yield(transport) if block_given?
+            end.to_transport(Core::Telemetry::Transport::Telemetry::Transport).tap do |transport|
+              transport.api_key = api_key
+            end
+          end
+
+          # Builds a new Transport::HTTP::Client with default settings
+          # Pass a block to override any settings.
+          def agent_telemetry(
+            agent_settings:,
+            logger:,
+            api_version: nil,
+            headers: nil
+          )
+            Core::Transport::HTTP.build(api_instance_class: Telemetry::API::Instance,
+              logger: logger,
+              agent_settings: agent_settings, api_version: api_version, headers: headers) do |transport|
+              apis = API.defaults
+
+              transport.api API::AGENT_TELEMETRY, apis[API::AGENT_TELEMETRY]
+
+              # Call block to apply any customization, if provided
+              yield(transport) if block_given?
+            end.to_transport(Core::Telemetry::Transport::Telemetry::Transport)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/transport/telemetry.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative '../../transport/parcel'
+require_relative 'http/client'
+require_relative 'http/telemetry'
+
+module Datadog
+  module Core
+    module Telemetry
+      module Transport
+        module Telemetry
+          class EncodedParcel
+            include Datadog::Core::Transport::Parcel
+          end
+
+          class Request < Datadog::Core::Transport::Request
+
+            attr_reader :request_type
+            attr_reader :api_key
+
+            def initialize(request_type, parcel, api_key)
+              @request_type = request_type
+              super(parcel)
+              @api_key = api_key
+            end
+          end
+
+          class Transport
+            attr_reader :client, :apis, :default_api, :current_api_id, :logger
+            attr_accessor :api_key
+
+            def initialize(apis, default_api, logger:)
+              @apis = apis
+              @logger = logger
+
+              @client = HTTP::Client.new(@apis[default_api], logger: logger)
+            end
+
+            def send_telemetry(request_type:, payload:)
+              json = JSON.dump(payload)
+              parcel = EncodedParcel.new(json)
+              request = Request.new(request_type, parcel, api_key)
+
+              @client.send_telemetry_payload(request)
+              # Perform no error checking here
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/worker.rb

@@ -52,10 +52,15 @@ module Datadog
 
         attr_reader :logger
 
+        # Returns true if worker thread is successfully started,
+        # false if worker thread was not started but telemetry is enabled,
+        # nil if telemetry is disabled.
         def start
           return if !enabled? || forked?
 
           # starts async worker
+          # perform should return true if thread was actually started,
+          # false otherwise
           perform
         end
 
@@ -65,10 +70,15 @@ module Datadog
           super
         end
 
+        # Returns true if event was enqueued, nil if not.
+        # While returning false may seem more reasonable, the only reason
+        # for not enqueueing event (presently) is that telemetry is disabled
+        # altogether, and in this case other methods return nil.
         def enqueue(event)
           return if !enabled? || forked?
 
           buffer.push(event)
+          true
         end
 
         def sent_started_event?
@@ -79,6 +89,39 @@ module Datadog
           TELEMETRY_STARTED_ONCE.failed?
         end
 
+        # Wait for the worker to send out all events that have already
+        # been queued, up to 15 seconds. Returns whether all events have
+        # been flushed.
+        #
+        # @api private
+        def flush
+          return true unless enabled? || !run_loop?
+
+          started = Utils::Time.get_time
+          loop do
+            # The AppStarted event is triggered by the worker itself,
+            # from the worker thread. As such the main thread has no way
+            # to delay itself until that event is queued and we need some
+            # way to wait until that event is sent out to assert on it in
+            # the test suite. Check the run once flag which *should*
+            # indicate the event has been queued (at which point our queue
+            # depth check should waint until it's sent).
+            # This is still a hack because the flag can be overridden
+            # either way with or without the event being sent out.
+            # Note that if the AppStarted sending fails, this check
+            # will return false and flushing will be blocked until the
+            # 15 second timeout.
+            # Note that the first wait interval between telemetry event
+            # sending is 10 seconds, the timeout needs to be strictly
+            # greater than that.
+            return true if buffer.empty? && !in_iteration? && TELEMETRY_STARTED_ONCE.success?
+
+            sleep 0.5
+
+            return false if Utils::Time.get_time - started > 15
+          end
+        end
+
         private
 
         def perform(*events)
@@ -99,6 +142,8 @@ module Datadog
 
         def flush_events(events)
           return if events.empty?
+          # TODO: can this method silently drop events which are
+          # generated prior to the started event being submitted?
           return if !enabled? || !sent_started_event?
 
           events = deduplicate_logs(events)

data/lib/datadog/core/utils/time.rb

@@ -31,6 +31,12 @@ module Datadog
         #
         # @param block [Proc] block that returns a `Time` object representing the current wall time
         def now_provider=(block)
+          class << self
+            # Avoid method redefinition warning.
+            # `rescue nil` is added in case customers remove the method
+            # themselves to squelch the warning.
+            remove_method(:now) rescue nil
+          end
           define_singleton_method(:now, &block)
         end
 
@@ -43,6 +49,12 @@ module Datadog
         #
         # @param block [Proc] block that accepts unit and returns timestamp in the requested unit
         def get_time_provider=(block)
+          class << self
+            # Avoid method redefinition warning
+            # `rescue nil` is added in case customers remove the method
+            # themselves to squelch the warning.
+            remove_method(:get_time) rescue nil
+          end
           define_singleton_method(:get_time, &block)
         end
 

data/lib/datadog/core/workers/async.rb

@@ -47,6 +47,14 @@ module Datadog
             @run_async = false
             Datadog.logger.debug { "Forcibly terminating worker thread for: #{self}" }
             worker.terminate
+            # Wait for the worker thread to end
+            begin
+              Timeout.timeout(SHUTDOWN_TIMEOUT) do
+                worker.join
+              end
+            rescue Timeout::Error
+              Datadog.logger.debug { "Worker thread did not end after #{SHUTDOWN_TIMEOUT} seconds: #{self}" }
+            end
             true
           end
 
@@ -112,23 +120,33 @@ module Datadog
             @worker ||= nil
           end
 
+          # Returns true if worker thread is successfully started,
+          # false if it is not started. Reasons for not starting the worker
+          # thread: it is already running, or the process forked and fork
+          # policy is to stop the workers on fork (which means they are
+          # not started in children, really).
           def start_async(&block)
             mutex.synchronize do
-              return if running?
+              return false if running?
 
               if forked?
                 case fork_policy
                 when FORK_POLICY_STOP
                   stop_fork
+                  false
                 when FORK_POLICY_RESTART
+                  # restart_after_fork should return true
                   restart_after_fork(&block)
                 end
               elsif !run_async?
+                # start_worker should return true
                 start_worker(&block)
               end
             end
           end
 
+          # Returns true if worker thread is successfully started,
+          # which should generally be always.
           def start_worker
             @run_async = true
             @pid = Process.pid
@@ -151,7 +169,7 @@ module Datadog
             @worker.name = self.class.name
             @worker.thread_variable_set(:fork_safe, true)
 
-            nil
+            true
           end
 
           def stop_fork

data/lib/datadog/core/workers/interval_loop.rb

@@ -21,7 +21,18 @@ module Datadog
         # Methods that must be prepended
         module PrependedMethods
           def perform(*args)
-            perform_loop { super(*args) }
+            perform_loop do
+              @in_iteration = true
+              begin
+                super(*args)
+              ensure
+                @in_iteration = false
+              end
+            end
+          end
+
+          def in_iteration?
+            defined?(@in_iteration) && @in_iteration
           end
         end
 

data/lib/datadog/core/workers/runtime_metrics.rb

@@ -20,8 +20,8 @@ module Datadog
         attr_reader \
           :metrics
 
-        def initialize(options = {})
-          @metrics = options.fetch(:metrics) { Core::Runtime::Metrics.new(logger: options[:logger]) }
+        def initialize(telemetry:, **options)
+          @metrics = options.fetch(:metrics) { Core::Runtime::Metrics.new(logger: options[:logger], telemetry: telemetry) }
 
           # Workers::Async::Thread settings
           self.fork_policy = options.fetch(:fork_policy, Workers::Async::Thread::FORK_POLICY_STOP)

data/lib/datadog/core.rb

@@ -11,6 +11,14 @@ module Datadog
   # for higher-level features.
   module Core
     extend Core::Deprecations
+
+    LIBDATADOG_API_FAILURE =
+      begin
+        require "libdatadog_api.#{RUBY_VERSION[/\d+.\d+/]}_#{RUBY_PLATFORM}"
+        nil
+      rescue LoadError => e
+        e.message
+      end
   end
 
   extend Core::Extensions