datadog 2.8.0 → 2.39.0

data/lib/datadog/di/transport/input.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require_relative "../../core/chunker"
+require_relative "../../core/encoding"
+require_relative "../../core/tag_builder"
+require_relative "../../core/transport/parcel"
+require_relative "../../core/transport/request"
+require_relative "../../core/transport/transport"
+require_relative "../error"
+require_relative "../fatal_exceptions"
+require_relative "http/input"
+
+module Datadog
+  module DI
+    module Transport
+      module Input
+        class Request < Datadog::Core::Transport::Request
+          attr_reader :serialized_tags
+
+          def initialize(parcel, serialized_tags)
+            super(parcel)
+
+            @serialized_tags = serialized_tags
+          end
+        end
+
+        class Transport < Core::Transport::Transport
+          attr_reader :telemetry
+
+          def initialize(apis, default_api, logger:, telemetry: nil)
+            super(apis, default_api, logger: logger)
+            @telemetry = telemetry
+          end
+
+          # The limit on an individual snapshot payload, aka "log line",
+          # is 1 MB.
+          #
+          # TODO There is an RFC for snapshot pruning that should be
+          # implemented to reduce the size of snapshots to be below this
+          # limit, so that we can send a portion of the captured data
+          # rather than dropping the snapshot entirely.
+          MAX_SERIALIZED_SNAPSHOT_SIZE = 1024 * 1024
+
+          # The maximum chunk (batch) size that intake permits is 5 MB.
+          #
+          # Two bytes are for the [ and ] of JSON array syntax.
+          MAX_CHUNK_SIZE = 5 * 1024 * 1024 - 2
+
+          # Try to send smaller payloads to avoid large network requests.
+          # If a payload is larger than default chunk size but is under the
+          # max chunk size, it will still get sent out.
+          DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024
+
+          # Sends snapshot payloads to the agent.
+          #
+          # Each snapshot is serialized individually. If serialization fails
+          # for a snapshot (e.g., due to binary data from custom serializers),
+          # the on_serialization_error callback is invoked with the probe ID
+          # and exception, allowing the caller to disable the affected probe.
+          # Successfully serialized snapshots are still sent.
+          #
+          # Large snapshots (> 1MB) are dropped. Batches are split into chunks
+          # of ~2MB each to avoid large network requests.
+          #
+          # @param payload [Array<Hash>] Array of snapshot payloads
+          # @param tags [Hash] Tags to send with the snapshots
+          # @param on_serialization_error [Proc] Called with (probe_id, exception)
+          #   when a snapshot fails to serialize.
+          def send_input(payload, tags, on_serialization_error:)
+            serialized_tags = Core::TagBuilder.serialize_tags(tags)
+
+            # Serialize each snapshot individually to isolate failures
+            encoded_snapshots = []
+            payload.each do |snapshot|
+              encoded = encoder.encode(snapshot)
+              if encoded.length > MAX_SERIALIZED_SNAPSHOT_SIZE
+                logger.debug { "di: dropping too big snapshot" }
+                next
+              end
+              encoded_snapshots << encoded
+            rescue Exception => exc # standard:disable Lint/RescueException
+              Datadog::DI.reraise_if_fatal(exc)
+              # Serialization failed for this snapshot - report via callback
+              # This catches JSON::GeneratorError, Encoding errors, TypeError, etc.
+              probe_id = snapshot.dig(:debugger, :snapshot, :probe, :id)
+              logger.debug { "di: JSON encoding failed for snapshot (probe #{probe_id}): #{exc.class}: #{exc.message}" }
+              telemetry&.report(exc, description: "JSON encoding failed for snapshot")
+
+              if probe_id
+                begin
+                  on_serialization_error.call(probe_id, exc)
+                rescue Exception => callback_exc # standard:disable Lint/RescueException
+                  Datadog::DI.reraise_if_fatal(callback_exc)
+                  logger.debug { "di: error in serialization error callback for probe #{probe_id}: #{callback_exc.class}: #{callback_exc.message}" }
+                  telemetry&.report(callback_exc, description: "Error in serialization error callback")
+                end
+              end
+            end
+
+            return payload if encoded_snapshots.empty?
+
+            Datadog::Core::Chunker.chunk_by_size(
+              encoded_snapshots, DEFAULT_CHUNK_SIZE,
+            ).each do |chunk|
+              # We drop snapshots that are too big earlier.
+              # The limit on chunked payload length here is greater
+              # than the limit on snapshot size, therefore no chunks
+              # can exceed limits here.
+              chunked_payload = encoder.join(chunk)
+
+              # We need to rescue exceptions for each chunk so that
+              # subsequent chunks are attempted to be sent.
+              begin
+                send_input_chunk(chunked_payload, serialized_tags)
+              rescue Exception => exc # standard:disable Lint/RescueException
+                Datadog::DI.reraise_if_fatal(exc)
+                logger.debug { "di: failed to send snapshot chunk: #{exc.class}: #{exc.message} (at #{exc.backtrace&.first})" }
+                telemetry&.report(exc, description: "Error sending snapshot chunk")
+              end
+            end
+
+            payload
+          end
+
+          def send_input_chunk(chunked_payload, serialized_tags)
+            parcel = Core::Transport::Parcel.new(chunked_payload, content_type: encoder.content_type)
+            request = Request.new(parcel, serialized_tags)
+
+            client.send_request(:input, request).tap do |response|
+              if downgrade?(response)
+                downgrade!
+                return send_input_chunk(chunked_payload, serialized_tags)
+              end
+            end
+          end
+
+          def encoder
+            Core::Encoding::JSONEncoder
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/utils.rb

@@ -1,17 +1,112 @@
 # frozen_string_literal: true
 
+# standard gets itself into an infinite loop over this
+# rubocop:disable Layout/SpaceAfterNot
+
 module Datadog
   module DI
     module Utils
+      # General path matching considerations
+      # ------------------------------------
+      #
+      # The following use cases must be supported:
+      # 1. The "probe path" is relative path to the file from source code
+      #    repository root. The project is deployed from the repository root,
+      #    such that that same relative path exists at runtime from the
+      #    root of the application.
+      # 2. The "probe path" is a relative path to the file in a monorepo
+      #    where the project being deployed is in a subdirectory.
+      #    This the "probe path" contains additional directory components
+      #    in the beginning that do not exist in the runtime environment.
+      # 3. The "probe path" is an absolute path to the file on the customer's
+      #    development system. As specified this path definitely does not
+      #    exist at runtime, and can start with a prefix that is unknown
+      #    to any both UI and tracer code.
+      # 4. Same as (3), but the customer is using a Windows computer for
+      #    development and has the path specified in the wrong case
+      #    (which works fine on their development machine).
+      # 5. The "probe path" is the basename or any suffix of the path to
+      #    the desired file, typed manually by the customer into the UI.
+      #
+      # A related concern is that if multiple runtime paths match the path
+      # specification in the probe, the tracer must return an error to the
+      # backend/UI rather than instrumenting any of the matching paths.
+      #
+      # The logic for path matching should therefore, generally, be as follows:
+      # 1. If the "probe path" is absolute, see if it exists at runtime.
+      #    If so, take it as the desired path and finish.
+      # 2. Attempt to identify the application root, by checking if the current
+      #    working directory contains a file called Gemfile. If yes, assume
+      #    the current working directory is the application root, otherwise
+      #    consider the application root to be unknown.
+      # 3. If the application root is known and the "probe path" is relative,
+      #    concatenate the "probe path" to the application root and check
+      #    if the resulting path exists at runtime. If so, take it as the
+      #    desired path and finish.
+      # 4. If the "probe path" is relative, go through the known file paths,
+      #    filter these paths down to those whose suffix is the "probe path",
+      #    and check how many we are left with. If exactly one, assume this
+      #    is the desired path and finish. If more than one, return an error
+      #    "multiple matching paths".
+      # 5. If the application root is known, for each suffix of the "probe path",
+      #    see if that relative paths concatenated to the application root
+      #    results in a known file. If a known file is found, assume this
+      #    is the wanted file and finish.
+      # 6. For each suffix of the "probe path", filter the set of known paths
+      #    down to those that end in the suffix. If exactly one path remains
+      #    for a given suffix, assume this is the wanted path and finish.
+      #    If more than one path remains for a given suffix, return the error
+      #    "multiple matching paths".
+      # 7. Repeat step 5 but perform case-insensitive comparison.
+      # 8. Repeat step 6 but perform case-insensitive comparison.
+      #
+      # Note that we do not look for "probe paths" under the current working
+      # directory at runtime if the current working directory does not contain
+      # a Gemfile, to avoid finding files from potentially undesired bases.
+      #
+      # What "known file"/"known path" means also differs based on the
+      # operation being performed:
+      # - If the operation is "install a probe", "known file/path" can
+      #   include files on the filesystem that have not been loaded as
+      #   well as paths from the code tracker.
+      # - If the operation is "check if any pending probes match the file
+      #   that was just loaded", we would only consider the path that was
+      #   just loaded and not check the filesystem.
+      #
+      # Filesystem inquiries are obviously quite expensive and should be
+      # cached. For the vast majority of applications it should be safe to
+      # indefinitely cache whether a particular filesystem paths exists
+      # in both positive and negative.
+      #
+      # As a "quick fix", currently after performing the suffix matching
+      # we just strip leading directory components from the "probe path"
+      # until we get a match via a "suffix of the suffix".
+
+      # Returns +path+ with Windows-style backslash separators translated to
+      # forward slashes (DEBUG-5111). Used to normalize probe source paths
+      # that originate from IDE tooling running on Windows.
+      module_function def normalize_windows_separators(path)
+        path.tr("\\", "/")
+      end
+
       # Returns whether the provided +path+ matches the user-designated
       # file suffix (of a line probe).
       #
-      # If suffix is an absolute path (i.e., it starts with a slash), the path
-      # must be identical for it to match.
+      # Backslash separators in +suffix+ are translated to forward slashes
+      # (DEBUG-5111) so paths typed by IDE tooling on Windows can match.
+      # Case-insensitive matching (DEBUG-5107) is opt-in via
+      # +case_insensitive: true+; callers that orchestrate matching against a
+      # set of known paths perform case-sensitive comparisons first and only
+      # fall back to case-insensitive when no case-sensitive match is found
+      # (see the design comment above, steps 5-8).
+      #
+      # If suffix is an absolute path (i.e., it starts with a slash, possibly
+      # after backslash normalization), the path must be identical for it to
+      # match.
       #
       # If suffix is not an absolute path, the path matches if its suffix is
       # the provided suffix, at a path component boundary.
-      module_function def path_matches_suffix?(path, suffix)
+      module_function def path_matches_suffix?(path, suffix, case_insensitive: false)
         if path.nil?
           raise ArgumentError, "nil path passed"
         end
@@ -19,18 +114,19 @@ module Datadog
           raise ArgumentError, "nil suffix passed"
         end
 
-        if suffix.start_with?('/')
+        suffix = normalize_windows_separators(suffix)
+        if case_insensitive
+          path = path.downcase
+          suffix = suffix.downcase
+        end
+
+        if suffix.start_with?("/")
           path == suffix
         else
           # Exact match is not possible here, meaning any matching path
           # has to be longer than the suffix. Require full component matches,
           # meaning either the first character of the suffix is a slash
           # or the previous character in the path is a slash.
-          # For now only check for forward slashes for Unix-like OSes;
-          # backslash is a legitimate character of a file name in Unix
-          # therefore simply permitting forward or back slash is not
-          # sufficient, we need to perform an OS check to know which
-          # path separator to use.
           !!
           if path.length > suffix.length && path.end_with?(suffix)
             previous_char = path[path.length - suffix.length - 1]
@@ -41,6 +137,34 @@ module Datadog
           # !!(path =~ %r,(/|\A)#{Regexp.quote(suffix)}\z,)
         end
       end
+
+      # Returns whether the provided +path+ matches the "probe path" in
+      # +spec+. Attempts all of the fuzzy matches by stripping directories
+      # from the front of +spec+. Does not consider othr known paths to
+      # identify the case of (potentially) multiple matching paths for +spec+.
+      #
+      # Matching is attempted case-sensitively first (steps 5-6 in the design
+      # comment above) and only falls back to case-insensitive (steps 7-8)
+      # when no case-sensitive match is found.
+      module_function def path_can_match_spec?(path, spec)
+        # Normalize Windows-style backslash separators (DEBUG-5111) so the
+        # suffix-shortening loop's "/+" regex can strip leading components.
+        spec = normalize_windows_separators(spec)
+
+        [false, true].each do |case_insensitive|
+          working_spec = spec.dup
+          return true if path_matches_suffix?(path, working_spec, case_insensitive: case_insensitive)
+
+          loop do
+            break unless working_spec.include?("/")
+            working_spec.sub!(%r{.*/+}, "")
+            return true if path_matches_suffix?(path, working_spec, case_insensitive: case_insensitive)
+          end
+        end
+        false
+      end
     end
   end
 end
+
+# rubocop:enable Layout/SpaceAfterNot