datadog 2.2.0 → 2.4.0

data/lib/datadog/core/telemetry/http/adapters/net.rb

@@ -34,19 +34,17 @@ module Datadog
             end
 
             def post(env)
-              begin
-                post = ::Net::HTTP::Post.new(env.path, env.headers)
-                post.body = env.body
-
-                http_response = open do |http|
-                  http.request(post)
-                end
-
-                Response.new(http_response)
-              rescue StandardError => e
-                Datadog.logger.debug("Unable to send telemetry event to agent: #{e}")
-                Telemetry::Http::InternalErrorResponse.new(e)
+              post = ::Net::HTTP::Post.new(env.path, env.headers)
+              post.body = env.body
+
+              http_response = open do |http|
+                http.request(post)
               end
+
+              Response.new(http_response)
+            rescue StandardError => e
+              Datadog.logger.debug("Unable to send telemetry event to agent: #{e}")
+              Telemetry::Http::InternalErrorResponse.new(e)
             end
 
             # Data structure for an HTTP Response

data/lib/datadog/core/telemetry/http/ext.rb

@@ -17,7 +17,10 @@ module Datadog
           CONTENT_TYPE_APPLICATION_JSON = 'application/json'
           API_VERSION = 'v2'
 
+          AGENTLESS_HOST_PREFIX = 'instrumentation-telemetry-intake'
+
           AGENT_ENDPOINT = '/telemetry/proxy/api/v2/apmtelemetry'
+          AGENTLESS_ENDPOINT = '/api/v2/apmtelemetry'
         end
       end
     end

data/lib/datadog/core/telemetry/http/transport.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../configuration/settings'
+require_relative '../../environment/ext'
 require_relative '../../transport/ext'
 require_relative 'env'
 require_relative 'ext'
@@ -13,18 +14,42 @@ module Datadog
         # Class to send telemetry data to Telemetry API
         # Currently only supports the HTTP protocol.
         class Transport
+          def self.build_agent_transport(agent_settings)
+            Transport.new(
+              host: agent_settings.hostname,
+              port: agent_settings.port,
+              path: Http::Ext::AGENT_ENDPOINT
+            )
+          end
+
+          def self.build_agentless_transport(api_key:, dd_site:, url_override: nil)
+            url = url_override || "https://#{Http::Ext::AGENTLESS_HOST_PREFIX}.#{dd_site}:443"
+
+            uri = URI.parse(url)
+            raise "Invalid agentless mode URL: #{url}" if uri.host.nil?
+
+            Transport.new(
+              host: uri.host,
+              port: uri.port || 80,
+              path: Http::Ext::AGENTLESS_ENDPOINT,
+              ssl: uri.scheme == 'https' || uri.port == 443,
+              api_key: api_key
+            )
+          end
+
           attr_reader \
             :host,
             :port,
             :ssl,
-            :path
-
-          def initialize
-            agent_settings = Configuration::AgentSettingsResolver.call(Datadog.configuration)
-            @host = agent_settings.hostname
-            @port = agent_settings.port
-            @ssl = false
-            @path = Http::Ext::AGENT_ENDPOINT
+            :path,
+            :api_key
+
+          def initialize(host:, port:, path:, ssl: false, api_key: nil)
+            @host = host
+            @port = port
+            @ssl = ssl
+            @path = path
+            @api_key = api_key
           end
 
           def request(request_type:, payload:)
@@ -38,7 +63,7 @@ module Datadog
           private
 
           def headers(request_type:, api_version: Http::Ext::API_VERSION)
-            {
+            result = {
               Core::Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST => '1',
               Ext::HEADER_CONTENT_TYPE => Http::Ext::CONTENT_TYPE_APPLICATION_JSON,
               Ext::HEADER_DD_TELEMETRY_API_VERSION => api_version,
@@ -49,6 +74,10 @@ module Datadog
               # Enable debug mode for telemetry
               # HEADER_TELEMETRY_DEBUG_ENABLED => 'true',
             }
+
+            result[Ext::HEADER_DD_API_KEY] = api_key unless api_key.nil?
+
+            result
           end
 
           def adapter

data/lib/datadog/core/telemetry/logger.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Telemetry
+      # === INTERNAL USAGE ONLY ===
+      #
+      # Report telemetry logs via delegating to the telemetry component instance via mutex.
+      #
+      # IMPORTANT: Invoking this method during the lifecycle of component initialization will
+      # be no-op instead.
+      #
+      # For developer using this module:
+      #   read: lib/datadog/core/telemetry/logging.rb
+      module Logger
+        class << self
+          def report(exception, level: :error, description: nil)
+            instance&.report(exception, level: level, description: description)
+          end
+
+          def error(description)
+            instance&.error(description)
+          end
+
+          private
+
+          def instance
+            # Component initialization uses a mutex to avoid having concurrent initialization.
+            # Trying to access the telemetry component during initialization (specifically:
+            # from the thread that's actually doing the initialization) would cause a deadlock,
+            # since accessing the components would try to recursively lock the mutex.
+            #
+            # To work around this, we use allow_initialization: false to avoid triggering this issue.
+            #
+            # The downside is: this leaves us unable to report telemetry during component initialization.
+            components = Datadog.send(:components, allow_initialization: false)
+
+            if components && components.telemetry
+              components.telemetry
+            else
+              Datadog.logger.warn(
+                'Failed to send telemetry before components initialization or within components lifecycle'
+              )
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/logging.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require_relative 'event'
+
+require 'pathname'
+
+module Datadog
+  module Core
+    module Telemetry
+      # === INTERNAL USAGE ONLY ===
+      #
+      # Logging interface for sending telemetry logs... so we can fix them.
+      #
+      # For developer using this module:
+      # - MUST NOT provide any sensitive information (PII)
+      # - SHOULD reduce the data cardinality for batching/aggregation
+      #
+      # Before using it, ask yourself:
+      # - Do we need to know about this (ie. internal error or client error)?
+      # - How severe/critical is this error? (ie. error, warning, fatal)
+      # - What information needed to make it actionable?
+      #
+      module Logging
+        # Extract datadog stack trace from the exception
+        module DatadogStackTrace
+          GEM_ROOT = Pathname.new("#{__dir__}/../../../..").cleanpath.to_s
+
+          def self.from(exception)
+            backtrace = exception.backtrace
+
+            return unless backtrace
+            return if backtrace.empty?
+
+            stack_trace = +''
+            backtrace.each do |line|
+              stack_trace << if line.start_with?(GEM_ROOT)
+                               line[GEM_ROOT.length..-1] || ''
+                             else
+                               'REDACTED'
+                             end
+              stack_trace << ','
+            end
+
+            stack_trace.chomp(',')
+          end
+        end
+
+        def report(exception, level: :error, description: nil)
+          # Annoymous exceptions to be logged as <Class:0x00007f8b1c0b3b40>
+          message = +''
+          message << (exception.class.name || exception.class.inspect)
+          message << ':' << description if description
+
+          event = Event::Log.new(
+            message: message,
+            level: level,
+            stack_trace: DatadogStackTrace.from(exception)
+          )
+
+          log!(event)
+        end
+
+        def error(description)
+          event = Event::Log.new(message: description, level: :error)
+
+          log!(event)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/telemetry/request.rb

@@ -31,6 +31,18 @@ module Datadog
 
           def application
             config = Datadog.configuration
+
+            tracer_version = Core::Environment::Identity.gem_datadog_version_semver2
+
+            # We need some to distinguish datadog-ci gem versions
+            # when examining telemetry metrics emitted from the datadog-ci gem.
+            #
+            # This code checks that Datadog::CI is loaded and ci mode is enabled and adds
+            # "-ci-X.Y.Z" suffix to the tracer version.
+            if defined?(::Datadog::CI::VERSION) && config.respond_to?(:ci) && config.ci.enabled
+              tracer_version = "#{tracer_version}-ci-#{::Datadog::CI::VERSION::STRING}"
+            end
+
             {
               env: config.env,
               language_name: Core::Environment::Ext::LANG,
@@ -39,7 +51,7 @@ module Datadog
               runtime_version: Core::Environment::Ext::ENGINE_VERSION,
               service_name: config.service,
               service_version: config.version,
-              tracer_version: Core::Environment::Identity.gem_datadog_version_semver2
+              tracer_version: tracer_version
             }
           end
 

data/lib/datadog/core/utils/at_fork_monkey_patch.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Utils
+      # Monkey patches `Kernel#fork` and similar functions, adding an `at_fork` callback mechanism which
+      # is used to restart observability after the VM forks (e.g. in multiprocess Ruby apps).
+      module AtForkMonkeyPatch
+        AT_FORK_CHILD_BLOCKS = [] # rubocop:disable Style/MutableConstant Used to store blocks to run, mutable by design.
+        private_constant :AT_FORK_CHILD_BLOCKS
+
+        def self.supported?
+          Process.respond_to?(:fork)
+        end
+
+        def self.apply!
+          return false unless supported?
+
+          if RUBY_VERSION < '3.1'
+            [
+              ::Process.singleton_class, # Process.fork
+              ::Kernel.singleton_class,  # Kernel.fork
+              ::Object,                  # fork without explicit receiver (it's defined as a method in ::Kernel)
+              # Note: Modifying Object as we do here is irreversible. During tests, this
+              # change will stick around even if we otherwise stub `Process` and `Kernel`
+            ].each { |target| target.prepend(KernelMonkeyPatch) }
+          end
+
+          ::Process.singleton_class.prepend(ProcessMonkeyPatch)
+
+          true
+        end
+
+        def self.run_at_fork_blocks(stage)
+          raise(ArgumentError, "Unsupported stage #{stage}") unless stage == :child
+
+          AT_FORK_CHILD_BLOCKS.each(&:call)
+        end
+
+        def self.at_fork(stage, &block)
+          raise(ArgumentError, "Unsupported stage #{stage}") unless stage == :child
+          raise(ArgumentError, 'Missing block argument') unless block
+
+          AT_FORK_CHILD_BLOCKS << block
+
+          true
+        end
+
+        # Adds `at_fork` behavior; see parent module for details.
+        module KernelMonkeyPatch
+          def fork
+            # If a block is provided, it must be wrapped to trigger callbacks.
+            child_block = if block_given?
+                            proc do
+                              AtForkMonkeyPatch.run_at_fork_blocks(:child)
+
+                              # Invoke original block
+                              yield
+                            end
+                          end
+
+            # Start fork
+            # If a block is provided, use the wrapped version.
+            result = child_block.nil? ? super : super(&child_block)
+
+            # When fork gets called without a block, it returns twice:
+            # If we're in the fork, result = nil: trigger child callbacks.
+            # If we're in the parent, result = pid: we do nothing.
+            # (If it gets called with a block, it only returns on the parent)
+            AtForkMonkeyPatch.run_at_fork_blocks(:child) if result.nil?
+
+            result
+          end
+        end
+
+        # Adds `at_fork` behavior; see parent module for details.
+        module ProcessMonkeyPatch
+          # Hook provided by Ruby 3.1+ for observability libraries that want to know about fork, see
+          # https://github.com/ruby/ruby/pull/5017 and https://bugs.ruby-lang.org/issues/17795
+          def _fork
+            pid = super
+
+            AtForkMonkeyPatch.run_at_fork_blocks(:child) if pid == 0
+
+            pid
+          end
+
+          # A call to Process.daemon ( https://rubyapi.org/3.1/o/process#method-c-daemon ) forks the current process and
+          # keeps executing code in the child process, killing off the parent, thus effectively replacing it.
+          # This is not covered by `_fork` and thus we have some extra code for it.
+          def daemon(*args)
+            result = super
+
+            AtForkMonkeyPatch.run_at_fork_blocks(:child)
+
+            result
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/utils/time.rb

@@ -34,6 +34,18 @@ module Datadog
           define_singleton_method(:now, &block)
         end
 
+        # Overrides the implementation of `#get_time
+        # with the provided callable.
+        #
+        # Overriding the method `#get_time` instead of
+        # indirectly calling `block` removes
+        # one level of method call overhead.
+        #
+        # @param block [Proc] block that accepts unit and returns timestamp in the requested unit
+        def get_time_provider=(block)
+          define_singleton_method(:get_time, &block)
+        end
+
         def measure(unit = :float_second)
           before = get_time(unit)
           yield

data/lib/datadog/di/code_tracker.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # Tracks loaded Ruby code by source file and maintains a map from
+    # source file to the loaded code (instruction sequences).
+    # Also arranges for code in the loaded files to be instrumented by
+    # line probes that have already been received by the library.
+    #
+    # The loaded code is used to target line trace points when installing
+    # line probes which dramatically improves efficiency of line trace points.
+    #
+    # Note that, since most files will only be loaded one time (via the
+    # "require" mechanism), the code tracker needs to be global and not be
+    # recreated when the DI component is created.
+    #
+    # @api private
+    class CodeTracker
+      def initialize
+        @registry = {}
+        @trace_point_lock = Mutex.new
+        @registry_lock = Mutex.new
+        @compiled_trace_point = nil
+      end
+
+      # Starts tracking loaded code.
+      #
+      # This method should generally be called early in application boot
+      # process, because any code loaded before code tracking is enabled
+      # will not be instrumentable via line probes.
+      #
+      # Normally tracking should remain active for the lifetime of the
+      # process and would not be ever stopped.
+      def start
+        trace_point_lock.synchronize do
+          # If this code tracker is already running, we can do nothing or
+          # restart it (by disabling the trace point and recreating it).
+          # It is likely that some applications will attempt to activate
+          # DI more than once where the intention is to just activate DI;
+          # do not break such applications by clearing out the registry.
+          # For now, until there is a use case for recreating the trace point,
+          # do nothing if the code tracker has already started.
+          return if @compiled_trace_point
+
+          # Note: .trace enables the trace point.
+          @compiled_trace_point = TracePoint.trace(:script_compiled) do |tp|
+            # Useful attributes of the trace point object here:
+            # .instruction_sequence
+            # .instruction_sequence.path (either absolute file path for
+            #   loaded or required code, or for eval'd code, if filename
+            #   is specified as argument to eval, then this is the provided
+            #   filename, otherwise this is a synthesized
+            #   "(eval at <definition-file>:<line>)" string)
+            # .instruction_sequence.absolute_path (absolute file path when
+            #   load or require are used to load code, nil for eval'd code
+            #   regardless of whether filename was specified as an argument
+            #   to eval on ruby 3.1+, same as path for eval'd code on ruby 3.0
+            #   and lower)
+            # .method_id
+            # .path (refers to the code location that called the require/eval/etc.,
+            #   not where the loaded code is; use .path on the instruction sequence
+            #   to obtain the location of the compiled code)
+            # .eval_script
+            #
+            # For now just map the path to the instruction sequence.
+            path = tp.instruction_sequence.absolute_path
+            # Do not store mapping for eval'd code, since there is no way
+            # to target such code from dynamic instrumentation UI.
+            # eval'd code always sets tp.eval_script.
+            # When tp.eval_script is nil, code is either 'load'ed or 'require'd.
+            # steep, of course, complains about indexing with +path+
+            # without checking that it is not nil, so here, maybe there is
+            # some situation where path would in fact be nil and
+            # steep would end up saving the day.
+            if path && !tp.eval_script
+              registry_lock.synchronize do
+                registry[path] = tp.instruction_sequence
+              end
+            end
+          end
+        end
+      end
+
+      # Returns whether this code tracker has been activated and is
+      # tracking.
+      def active?
+        trace_point_lock.synchronize do
+          !!@compiled_trace_point
+        end
+      end
+
+      # Returns an array of RubVM::InstructionSequence (i.e. the compiled code)
+      # for the provided path.
+      #
+      # The argument can be a full path to a Ruby source code file or a
+      # suffix (basename + one or more directories preceding the basename).
+      # The idea with suffix matches is that file paths are likely to
+      # be different between development and production environments and
+      # the source control system uses relative paths and doesn't have
+      # absolute paths at all.
+      #
+      # Suffix matches are not guaranteed to be correct, meaning there may
+      # be multiple files with the same basename and they may all match a
+      # given suffix. In such cases, this method will return all matching
+      # paths (and all of these paths will be attempted to be instrumented
+      # by upstream code).
+      #
+      # If the suffix matches one of the paths completely (which requires it
+      # to be an absolute path), only the exactly matching path is returned.
+      # Otherwise all known paths that end in the suffix are returned.
+      # If no paths match, an empty array is returned.
+      def iseqs_for_path(suffix)
+        registry_lock.synchronize do
+          exact = registry[suffix]
+          return [exact] if exact
+
+          inexact = []
+          registry.each do |path, iseq|
+            # Exact match is not possible here, meaning any matching path
+            # has to be longer than the suffix. Require full component matches,
+            # meaning either the first character of the suffix is a slash
+            # or the previous character in the path is a slash.
+            # For now only check for forward slashes for Unix-like OSes;
+            # backslash is a legitimate character of a file name in Unix
+            # therefore simply permitting forward or back slash is not
+            # sufficient, we need to perform an OS check to know which
+            # path separator to use.
+            if path.length > suffix.length && path.end_with?(suffix)
+              previous_char = path[path.length - suffix.length - 1]
+              inexact << iseq if previous_char == "/" || suffix[0] == "/"
+            end
+          end
+          inexact
+        end
+      end
+
+      # Stops tracking code that is being loaded.
+      #
+      # This method should ordinarily never be called - if a file is loaded
+      # when code tracking is not active, this file will not be instrumentable
+      # by line probes.
+      #
+      # This method is intended for test suite use only, where multiple
+      # code tracker instances are created, to fully clean up the old instances.
+      def stop
+        # Permit multiple stop calls.
+        trace_point_lock.synchronize do
+          @compiled_trace_point&.disable
+          # Clear the instance variable so that the trace point may be
+          # reinstated in the future.
+          @compiled_trace_point = nil
+        end
+        registry_lock.synchronize do
+          registry.clear
+        end
+      end
+
+      private
+
+      # Mapping from paths of loaded files to RubyVM::InstructionSequence
+      # objects representing compiled code of those files.
+      attr_reader :registry
+
+      attr_reader :trace_point_lock
+      attr_reader :registry_lock
+    end
+  end
+end