datadog 2.7.1 → 2.17.0

data/lib/datadog/di/transport/http/diagnostics.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/transport/http/api/instance'
+require_relative '../../../core/transport/http/api/spec'
+require_relative 'client'
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        module Diagnostics
+          module Client
+            def send_diagnostics_payload(request)
+              send_request(request) do |api, env|
+                api.send_diagnostics(env)
+              end
+            end
+          end
+
+          module API
+            class Instance < Core::Transport::HTTP::API::Instance
+              def send_diagnostics(env)
+                raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new('diagnostics', self) unless spec.is_a?(Diagnostics::API::Spec)
+
+                spec.send_diagnostics(env) do |request_env|
+                  call(request_env)
+                end
+              end
+            end
+
+            class Spec < Core::Transport::HTTP::API::Spec
+              attr_accessor :diagnostics
+
+              def send_diagnostics(env, &block)
+                raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('diagnostics', self) if diagnostics.nil?
+
+                diagnostics.call(env, &block)
+              end
+            end
+
+            class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
+              attr_reader :encoder
+
+              def initialize(path, encoder)
+                super(:post, path)
+                @encoder = encoder
+              end
+
+              def call(env, &block)
+                event_payload = Core::Vendor::Multipart::Post::UploadIO.new(
+                  StringIO.new(env.request.parcel.data), 'application/json', 'event.json'
+                )
+                env.form = {'event' => event_payload}
+
+                super
+              end
+            end
+          end
+        end
+
+        HTTP::Client.include(Diagnostics::Client)
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http/input.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/transport/http/api/instance'
+require_relative '../../../core/transport/http/api/spec'
+require_relative 'client'
+
+module Datadog
+  module DI
+    module Transport
+      module HTTP
+        module Input
+          module Client
+            def send_input_payload(request)
+              send_request(request) do |api, env|
+                api.send_input(env)
+              end
+            end
+          end
+
+          module API
+            class Instance < Core::Transport::HTTP::API::Instance
+              def send_input(env)
+                raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new('input', self) unless spec.is_a?(Input::API::Spec)
+
+                spec.send_input(env) do |request_env|
+                  call(request_env)
+                end
+              end
+            end
+
+            class Spec < Core::Transport::HTTP::API::Spec
+              attr_accessor :input
+
+              def send_input(env, &block)
+                raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('input', self) if input.nil?
+
+                input.call(env, &block)
+              end
+            end
+
+            class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
+              HEADER_CONTENT_TYPE = 'Content-Type'
+
+              attr_reader \
+                :encoder
+
+              def initialize(path, encoder)
+                super(:post, path)
+                @encoder = encoder
+              end
+
+              def call(env, &block)
+                # Encode body & type
+                env.headers[HEADER_CONTENT_TYPE] = encoder.content_type
+                env.body = env.request.parcel.data
+
+                super
+              end
+            end
+          end
+        end
+
+        HTTP::Client.include(Input::Client)
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/http.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative 'diagnostics'
+require_relative 'input'
+require_relative 'http/api'
+require_relative '../../core/transport/http'
+
+module Datadog
+  module DI
+    module Transport
+      # Namespace for HTTP transport components
+      module HTTP
+        module_function
+
+        # Builds a new Transport::HTTP::Client with default settings
+        # Pass a block to override any settings.
+        def diagnostics(
+          agent_settings:,
+          logger:,
+          api_version: nil,
+          headers: nil
+        )
+          Core::Transport::HTTP.build(api_instance_class: Diagnostics::API::Instance,
+            logger: logger,
+            agent_settings: agent_settings, api_version: api_version, headers: headers) do |transport|
+            apis = API.defaults
+
+            transport.api API::DIAGNOSTICS, apis[API::DIAGNOSTICS]
+
+            # Call block to apply any customization, if provided
+            yield(transport) if block_given?
+          end.to_transport(DI::Transport::Diagnostics::Transport)
+        end
+
+        # Builds a new Transport::HTTP::Client with default settings
+        # Pass a block to override any settings.
+        def input(
+          agent_settings:,
+          logger:,
+          api_version: nil,
+          headers: nil
+        )
+          Core::Transport::HTTP.build(api_instance_class: Input::API::Instance,
+            logger: logger,
+            agent_settings: agent_settings, api_version: api_version, headers: headers) do |transport|
+            apis = API.defaults
+
+            transport.api API::INPUT, apis[API::INPUT]
+
+            # Call block to apply any customization, if provided
+            yield(transport) if block_given?
+          end.to_transport(DI::Transport::Input::Transport)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/input.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative '../../core/transport/parcel'
+require_relative 'http/client'
+
+module Datadog
+  module DI
+    module Transport
+      module Input
+        class EncodedParcel
+          include Datadog::Core::Transport::Parcel
+        end
+
+        class Request < Datadog::Core::Transport::Request
+        end
+
+        class Transport
+          attr_reader :client, :apis, :default_api, :current_api_id, :logger
+
+          def initialize(apis, default_api, logger:)
+            @apis = apis
+            @logger = logger
+
+            @client = HTTP::Client.new(current_api, logger: logger)
+          end
+
+          def current_api
+            @apis[HTTP::API::INPUT]
+          end
+
+          def send_input(payload)
+            json = JSON.dump(payload)
+            parcel = EncodedParcel.new(json)
+            request = Request.new(parcel)
+
+            response = @client.send_input_payload(request)
+            unless response.ok?
+              # TODO Datadog::Core::Transport::InternalErrorResponse
+              # does not have +code+ method, what is the actual API of
+              # these response objects?
+              raise Error::AgentCommunicationError, "send_input failed: #{begin
+                response.code
+              rescue
+                "???"
+              end}: #{response.payload}"
+            end
+          rescue Error::AgentCommunicationError
+            raise
+          # Datadog::Core::Transport does not perform any exception mapping,
+          # therefore we could have any exception here from failure to parse
+          # agent URI for example.
+          # If we ever implement retries for network errors, we should distinguish
+          # actual network errors from non-network errors that are raised by
+          # transport code.
+          rescue => exc
+            raise Error::AgentCommunicationError, "send_input failed: #{exc.class}: #{exc}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/utils.rb

@@ -1,8 +1,87 @@
 # frozen_string_literal: true
 
+# standard gets itself into an infinite loop over this
+# rubocop:disable Layout/SpaceAfterNot
+
 module Datadog
   module DI
     module Utils
+      # General path matching considerations
+      # ------------------------------------
+      #
+      # The following use cases must be supported:
+      # 1. The "probe path" is relative path to the file from source code
+      #    repository root. The project is deployed from the repository root,
+      #    such that that same relative path exists at runtime from the
+      #    root of the application.
+      # 2. The "probe path" is a relative path to the file in a monorepo
+      #    where the project being deployed is in a subdirectory.
+      #    This the "probe path" contains additional directory components
+      #    in the beginning that do not exist in the runtime environment.
+      # 3. The "probe path" is an absolute path to the file on the customer's
+      #    development system. As specified this path definitely does not
+      #    exist at runtime, and can start with a prefix that is unknown
+      #    to any both UI and tracer code.
+      # 4. Same as (3), but the customer is using a Windows computer for
+      #    development and has the path specified in the wrong case
+      #    (which works fine on their development machine).
+      # 5. The "probe path" is the basename or any suffix of the path to
+      #    the desired file, typed manually by the customer into the UI.
+      #
+      # A related concern is that if multiple runtime paths match the path
+      # specification in the probe, the tracer must return an error to the
+      # backend/UI rather than instrumenting any of the matching paths.
+      #
+      # The logic for path matching should therefore, generally, be as follows:
+      # 1. If the "probe path" is absolute, see if it exists at runtime.
+      #    If so, take it as the desired path and finish.
+      # 2. Attempt to identify the application root, by checking if the current
+      #    working directory contains a file called Gemfile. If yes, assume
+      #    the current working directory is the application root, otherwise
+      #    consider the application root to be unknown.
+      # 3. If the application root is known and the "probe path" is relative,
+      #    concatenate the "probe path" to the application root and check
+      #    if the resulting path exists at runtime. If so, take it as the
+      #    desired path and finish.
+      # 4. If the "probe path" is relative, go through the known file paths,
+      #    filter these paths down to those whose suffix is the "probe path",
+      #    and check how many we are left with. If exactly one, assume this
+      #    is the desired path and finish. If more than one, return an error
+      #    "multiple matching paths".
+      # 5. If the application root is known, for each suffix of the "probe path",
+      #    see if that relative paths concatenated to the application root
+      #    results in a known file. If a known file is found, assume this
+      #    is the wanted file and finish.
+      # 6. For each suffix of the "probe path", filter the set of known paths
+      #    down to those that end in the suffix. If exactly one path remains
+      #    for a given suffix, assume this is the wanted path and finish.
+      #    If more than one path remains for a given suffix, return the error
+      #    "multiple matching paths".
+      # 7. Repeat step 5 but perform case-insensitive comparison.
+      # 8. Repeat step 6 but perform case-insensitive comparison.
+      #
+      # Note that we do not look for "probe paths" under the current working
+      # directory at runtime if the current working directory does not contain
+      # a Gemfile, to avoid finding files from potentially undesired bases.
+      #
+      # What "known file"/"known path" means also differs based on the
+      # operation being performed:
+      # - If the operation is "install a probe", "known file/path" can
+      #   include files on the filesystem that have not been loaded as
+      #   well as paths from the code tracker.
+      # - If the operation is "check if any pending probes match the file
+      #   that was just loaded", we would only consider the path that was
+      #   just loaded and not check the filesystem.
+      #
+      # Filesystem inquiries are obviously quite expensive and should be
+      # cached. For the vast majority of applications it should be safe to
+      # indefinitely cache whether a particular filesystem paths exists
+      # in both positive and negative.
+      #
+      # As a "quick fix", currently after performing the suffix matching
+      # we just strip leading directory components from the "probe path"
+      # until we get a match via a "suffix of the suffix".
+
       # Returns whether the provided +path+ matches the user-designated
       # file suffix (of a line probe).
       #
@@ -12,6 +91,13 @@ module Datadog
       # If suffix is not an absolute path, the path matches if its suffix is
       # the provided suffix, at a path component boundary.
       module_function def path_matches_suffix?(path, suffix)
+        if path.nil?
+          raise ArgumentError, "nil path passed"
+        end
+        if suffix.nil?
+          raise ArgumentError, "nil suffix passed"
+        end
+
         if suffix.start_with?('/')
           path == suffix
         else
@@ -34,6 +120,23 @@ module Datadog
           # !!(path =~ %r,(/|\A)#{Regexp.quote(suffix)}\z,)
         end
       end
+
+      # Returns whether the provided +path+ matches the "probe path" in
+      # +spec+. Attempts all of the fuzzy matches by stripping directories
+      # from the front of +spec+. Does not consider othr known paths to
+      # identify the case of (potentially) multiple matching paths for +spec+.
+      module_function def path_can_match_spec?(path, spec)
+        return true if path_matches_suffix?(path, spec)
+
+        spec = spec.dup
+        loop do
+          return false unless spec.include?('/')
+          spec.sub!(%r{.*/+}, '')
+          return true if path_matches_suffix?(path, spec)
+        end
+      end
     end
   end
 end
+
+# rubocop:enable Layout/SpaceAfterNot

data/lib/datadog/di.rb

@@ -1,32 +1,8 @@
 # frozen_string_literal: true
 
-require_relative 'di/error'
-require_relative 'di/code_tracker'
-require_relative 'di/component'
 require_relative 'di/configuration'
 require_relative 'di/extensions'
-require_relative 'di/instrumenter'
-require_relative 'di/probe'
-require_relative 'di/probe_builder'
-require_relative 'di/probe_manager'
-require_relative 'di/probe_notification_builder'
-require_relative 'di/probe_notifier_worker'
-require_relative 'di/redactor'
-require_relative 'di/serializer'
-require_relative 'di/transport'
-require_relative 'di/utils'
-
-if defined?(ActiveRecord::Base)
-  # The third-party library integrations need to be loaded after the
-  # third-party libraries are loaded. Tracing and appsec use Railtie
-  # to delay integrations until all of the application's dependencies
-  # are loaded, when running under Rails. We should do the same here in
-  # principle, however DI currently only has an ActiveRecord integration
-  # and AR should be loaded before any application code is loaded, being
-  # part of Rails, therefore for now we should be OK to just require the
-  # AR integration from here.
-  require_relative 'di/contrib/active_record'
-end
+require_relative 'di/remote'
 
 module Datadog
   # Namespace for Datadog dynamic instrumentation.
@@ -43,63 +19,25 @@ module Datadog
     Extensions.activate!
 
     class << self
-      attr_reader :code_tracker
 
-      # Activates code tracking. Normally this method should be called
-      # when the application starts. If instrumenting third-party code,
-      # code tracking needs to be enabled before the third-party libraries
-      # are loaded. If you definitely will not be instrumenting
-      # third-party libraries, activating tracking after third-party libraries
-      # have been loaded may improve lookup performance.
+      # This method is called from DI Remote handler to issue DI operations
+      # to the probe manager (add or remove probes).
       #
-      # TODO test that activating tracker multiple times preserves
-      # existing mappings in the registry
-      def activate_tracking!
-        (@code_tracker ||= CodeTracker.new).start
-      end
-
-      # Deactivates code tracking. In normal usage of DI this method should
-      # never be called, however it is used by DI's test suite to reset
-      # state for individual tests.
+      # When DI Remote is executing, Datadog.components should be initialized
+      # and we should be able to reference it to get to the DI component.
       #
-      # Note that deactivating tracking clears out the registry, losing
-      # the ability to look up files that have been loaded into the process
-      # already.
-      def deactivate_tracking!
-        code_tracker&.stop
-      end
-
-      # Returns whether code tracking is available.
-      # This method should be used instead of querying #code_tracker
-      # because the latter one may be nil.
-      def code_tracking_active?
-        code_tracker&.active? || false
-      end
-
+      # Given that we need the current_component anyway for code tracker,
+      # perhaps we should delete the +component+ method and just use
+      # +current_component+ in all cases.
       def component
-        # TODO uncomment when remote is merged
-        #Datadog.send(:components).dynamic_instrumentation
+        Datadog.send(:components).dynamic_instrumentation
       end
     end
   end
 end
 
-=begin not yet enabled
-# :script_compiled trace point was added in Ruby 2.6.
-if RUBY_VERSION >= '2.6'
-  begin
-    # Activate code tracking by default because line trace points will not work
-    # without it.
-    Datadog::DI.activate_tracking!
-  rescue => exc
-    if defined?(Datadog.logger)
-      Datadog.logger.warn("Failed to activate code tracking for DI: #{exc.class}: #{exc}")
-    else
-      # We do not have Datadog logger potentially because DI code tracker is
-      # being loaded early in application boot process and the rest of datadog
-      # wasn't loaded yet. Output to standard error.
-      warn("Failed to activate code tracking for DI: #{exc.class}: #{exc}")
-    end
-  end
-end
-=end
+# Line probes will not work on Ruby < 2.6 because of lack of :script_compiled
+# trace point. Activate DI automatically on supported Ruby versions but
+# always load its settings so that, for example, turning DI off when
+# we are on Ruby 2.5 does not produce exceptions.
+require_relative 'di/boot' if RUBY_VERSION >= '2.6' && RUBY_ENGINE != 'jruby'

data/lib/datadog/error_tracking/collector.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+
+module Datadog
+  module ErrorTracking
+    # The Collector is in charge, for a SpanOperation of storing the span events
+    # created when an error is handled. Each SpanOperation has a Collector as soon
+    # as a span event is created and the Collector has the same life time as the SpanOp.
+    #
+    # If an error is handled then rethrown, the SpanEvent corresponding to the error
+    # will be deleted. That is why we do not add directly the SpanEvent to the SpanOp.
+    #
+    # @api private
+    class Collector
+      SPAN_EVENTS_LIMIT = 100
+      LOCK = Mutex.new
+      # Proc called when the span_operation :after_stop event is published
+      def self.after_stop
+        @after_stop ||= proc do |span_op, error|
+          # if this proc is called, we are sure that span_op has a collector
+          collector = span_op.get_collector_or_initialize
+          # if an error exited the scope of the span, we delete the corresponding SpanEvent.
+          collector.on_error(span_op, error) if error
+
+          span_events = collector.span_events
+          span_op.span_events.concat(span_events)
+        end
+      end
+
+      def initialize
+        @span_event_per_error = {}
+      end
+
+      def add_span_event(span_op, span_event, error)
+        # When this is the first time we add a span event for a span,
+        # we suscribe to the :after_stop event
+        if @span_event_per_error.empty?
+          events = span_op.send(:events)
+          events.after_stop.subscribe(&self.class.after_stop)
+
+          # This tag is used by the Error Tracking product to report
+          # the error in Error Tracking
+          span_op.set_tag(Ext::SPAN_EVENTS_HAS_EXCEPTION, true)
+        end
+        # Set a limit to the number of span event we can store per SpanOp
+        # If an error has been handled several times in the same span we can still
+        # modify the event (even if the capacity is reached) in order to report
+        # the information of the last rescue
+        if @span_event_per_error.key?(error) || @span_event_per_error.length < SPAN_EVENTS_LIMIT
+          @span_event_per_error[error] = span_event
+        end
+      end
+
+      if RUBY_VERSION >= Ext::RUBY_VERSION_WITH_RESCUE_EVENT
+        # Starting from ruby3.3, as we are listening to :rescue event,
+        # we just want to remove the span event if the error was
+        # previously handled
+        def on_error(_span_op, error)
+          @span_event_per_error.delete(error)
+        end
+      else
+        # Up to ruby3.2, we are listening to :raise event. We need to ensure
+        # that an error exiting the scope of a span is not handled in a parent span.
+        # This function will propagate the span event to the parent span. If the
+        # error is not handled in the parent span, it will be deleted by design.
+        def on_error(span_op, error)
+          return unless @span_event_per_error.key?(error)
+
+          unless span_op.root?
+            parent = span_op.send(:parent)
+            LOCK.synchronize do
+              parent_collector = parent.get_collector_or_initialize { Collector.new }
+              parent_collector.add_span_event(parent, @span_event_per_error[error], error)
+            end
+          end
+
+          @span_event_per_error.delete(error)
+        end
+      end
+
+      def span_events
+        @span_event_per_error.values
+      end
+    end
+  end
+end