datadog 2.17.0 → 2.37.0

data/lib/datadog/data_streams/transport/http/stats.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative '../stats'
+require_relative '../../../core/transport/http/api/endpoint'
+require_relative '../../../core/transport/http/response'
+
+module Datadog
+  module DataStreams
+    module Transport
+      module HTTP
+        # HTTP transport behavior for Data Streams stats
+        module Stats
+          # Response from HTTP transport for DSM stats
+          class Response
+            include Datadog::Core::Transport::HTTP::Response
+
+            def initialize(http_response)
+              super
+            end
+          end
+
+          module API
+            # Endpoint for submitting DSM stats data
+            class Endpoint < Core::Transport::HTTP::API::Endpoint
+              def initialize(path)
+                super(:post, path)
+              end
+
+              def call(env, &block)
+                # Build request
+                env.verb = verb
+                env.path = path
+                env.body = env.request.parcel.data
+                if (content_type = env.request.parcel.content_type)
+                  env.headers['content-type'] = content_type
+                end
+                if (content_encoding = env.request.parcel.content_encoding)
+                  env.headers['content-encoding'] = content_encoding
+                end
+
+                # Send request
+                http_response = yield(env)
+
+                # Build response
+                Response.new(http_response)
+              end
+
+              def encoder
+                # DSM handles encoding in the transport layer
+                nil
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams/transport/http.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative '../../core/transport/http'
+require_relative 'http/stats'
+require_relative 'stats'
+
+module Datadog
+  module DataStreams
+    module Transport
+      # HTTP transport for Data Streams Monitoring
+      module HTTP
+        V01 = Stats::API::Endpoint.new(
+          '/v0.1/pipeline_stats'
+        )
+
+        module_function
+
+        # Builds a new Transport::HTTP::Client with default settings
+        def default(
+          agent_settings:,
+          logger:
+        )
+          Core::Transport::HTTP.build(
+            agent_settings: agent_settings,
+            logger: logger,
+          ) do |transport|
+            transport.api 'v0.1', V01, default: true
+
+            # Call block to apply any customization, if provided
+            yield(transport) if block_given?
+          end.to_transport(Transport::Stats::Transport)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams/transport/stats.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'msgpack'
+require 'zlib'
+require_relative '../../core/transport/parcel'
+require_relative '../../core/transport/request'
+require_relative '../../core/transport/transport'
+
+module Datadog
+  module DataStreams
+    module Transport
+      module Stats
+        # Request for DSM stats
+        class Request < Datadog::Core::Transport::Request
+        end
+
+        # Transport for Data Streams Monitoring stats
+        class Transport < Core::Transport::Transport
+          def send_stats(payload)
+            # MessagePack encode and gzip compress the payload
+            msgpack_data = MessagePack.pack(payload)
+            compressed_data = Zlib.gzip(msgpack_data)
+
+            # Create parcel and request
+            parcel = Core::Transport::Parcel.new(
+              compressed_data,
+              content_type: 'application/msgpack',
+              content_encoding: 'gzip',
+            )
+            request = Request.new(parcel)
+
+            # Send to agent
+            client.send_request(:stats, request)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require_relative 'data_streams/processor'
+require_relative 'data_streams/pathway_context'
+require_relative 'data_streams/configuration'
+require_relative 'data_streams/extensions'
+require_relative 'core/utils/time'
+
+module Datadog
+  # Datadog Data Streams Monitoring public API.
+  #
+  # The Datadog team ensures that public methods in this module
+  # only receive backwards compatible changes, and breaking changes
+  # will only occur in new major versions releases.
+  # @public_api
+  module DataStreams
+    class << self
+      # Set a produce checkpoint for Data Streams Monitoring
+      #
+      # @param type [String] The type of the checkpoint (e.g., 'kafka', 'kinesis', 'sns')
+      # @param destination [String] The destination (e.g., topic, exchange, stream name)
+      # @param auto_instrumentation [Boolean] Whether this checkpoint was set by auto-instrumentation (default: false)
+      # @param tags [Hash] Additional tags to include
+      # @yield [key, value] Block to inject context into carrier
+      # @return [String, nil] Base64 encoded pathway context or nil if disabled
+      # @public_api
+      def set_produce_checkpoint(type:, destination:, auto_instrumentation: false, tags: {}, &block)
+        processor&.set_produce_checkpoint(
+          type: type,
+          destination: destination,
+          manual_checkpoint: !auto_instrumentation,
+          tags: tags,
+          &block
+        )
+      end
+
+      # Set a consume checkpoint for Data Streams Monitoring
+      #
+      # @param type [String] The type of the checkpoint (e.g., 'kafka', 'kinesis', 'sns')
+      # @param source [String] The source (e.g., topic, exchange, stream name)
+      # @param auto_instrumentation [Boolean] Whether this checkpoint was set by auto-instrumentation (default: false)
+      # @param tags [Hash] Additional tags to include
+      # @yield [key] Block to extract context from carrier
+      # @return [String, nil] Base64 encoded pathway context or nil if disabled
+      # @public_api
+      def set_consume_checkpoint(type:, source:, auto_instrumentation: false, tags: {}, &block)
+        processor&.set_consume_checkpoint(
+          type: type,
+          source: source,
+          manual_checkpoint: !auto_instrumentation,
+          tags: tags,
+          &block
+        )
+      end
+
+      # Track Kafka produce offset for lag monitoring
+      #
+      # @param topic [String] The Kafka topic name
+      # @param partition [Integer] The partition number
+      # @param offset [Integer] The offset of the produced message
+      # @return [Boolean, nil] true if tracking succeeded, nil if disabled
+      # @!visibility private
+      def track_kafka_produce(topic, partition, offset)
+        processor&.track_kafka_produce(topic, partition, offset, Core::Utils::Time.now)
+      end
+
+      # Track Kafka message consumption for consumer lag monitoring
+      #
+      # @param topic [String] The Kafka topic name
+      # @param partition [Integer] The partition number
+      # @param offset [Integer] The offset of the consumed message
+      # @return [Boolean, nil] true if tracking succeeded, nil if disabled
+      # @!visibility private
+      def track_kafka_consume(topic, partition, offset)
+        processor&.track_kafka_consume(topic, partition, offset, Core::Utils::Time.now)
+      end
+
+      # Check if Data Streams Monitoring is enabled and available
+      #
+      # @return [Boolean] true if the processor is available
+      # @public_api
+      def enabled?
+        !processor.nil?
+      end
+
+      private
+
+      def processor
+        components.data_streams
+      end
+
+      def components
+        Datadog.send(:components)
+      end
+    end
+
+    # Expose Data Streams to global shared objects
+    Extensions.activate!
+  end
+end

data/lib/datadog/di/base.rb

@@ -8,8 +8,12 @@
 # are loaded, and also none of the rest of datadog library which also
 # has contrib code in other products.
 
+require_relative 'fatal_exceptions'
 require_relative 'code_tracker'
 
+# Needed since this file can be loaded without core
+require_relative '../ruby_version'
+
 module Datadog
   # Namespace for Datadog dynamic instrumentation.
   #
@@ -17,6 +21,10 @@ module Datadog
   module DI
     LOCK = Mutex.new
 
+    # Initialized eagerly to avoid "instance variable not initialized"
+    # warning on Ruby 2.6/2.7 and to simplify the type to non-nullable.
+    @current_components = []
+
     class << self
       attr_reader :code_tracker
 
@@ -40,20 +48,21 @@ module Datadog
       # DI code.
       def activate_tracking
         # :script_compiled trace point was added in Ruby 2.6.
-        return unless RUBY_VERSION >= '2.6'
+        return unless RubyVersion.is?('>= 2.6')
 
         begin
           # Activate code tracking by default because line trace points will not work
           # without it.
           Datadog::DI.activate_tracking!
-        rescue => exc
+        rescue Exception => exc # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(exc)
           if defined?(Datadog.logger)
-            Datadog.logger.warn { "di: Failed to activate code tracking for DI: #{exc.class}: #{exc}" }
+            Datadog.logger.warn { "di: Failed to activate code tracking for DI: #{exc.class}: #{exc.message}" }
           else
             # We do not have Datadog logger potentially because DI code tracker is
             # being loaded early in application boot process and the rest of datadog
             # wasn't loaded yet. Output to standard error.
-            warn("datadog: di: Failed to activate code tracking for DI: #{exc.class}: #{exc}")
+            warn("datadog: di: Failed to activate code tracking for DI: #{exc.class}: #{exc.message}")
           end
         end
       end
@@ -88,7 +97,7 @@ module Datadog
       # Datadog.components from the code tracker.
       def current_component
         LOCK.synchronize do
-          @current_components&.last
+          @current_components.last
         end
       end
 
@@ -100,14 +109,13 @@ module Datadog
       # guaranteed to not end up with no component when one is running.
       def add_current_component(component)
         LOCK.synchronize do
-          @current_components ||= []
           @current_components << component
         end
       end
 
       def remove_current_component(component)
         LOCK.synchronize do
-          @current_components&.delete(component)
+          @current_components.delete(component)
         end
       end
     end

data/lib/datadog/di/boot.rb

@@ -5,18 +5,21 @@ require_relative 'base'
 require_relative 'error'
 require_relative 'code_tracker'
 require_relative 'component'
+require_relative 'context'
 require_relative 'instrumenter'
 require_relative 'probe'
 require_relative 'probe_builder'
 require_relative 'probe_manager'
 require_relative 'probe_notification_builder'
 require_relative 'probe_notifier_worker'
+require_relative 'probe_repository'
 require_relative 'redactor'
 require_relative 'serializer'
 require_relative 'transport/http'
 require_relative 'utils'
 
-if %w[1 true yes].include?(ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore
+if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED'])
+
   # For initial release of Dynamic Instrumentation, activate code tracking
   # only if DI is explicitly requested in the environment.
   # Code tracking is required for line probes to work; see the comments
@@ -32,3 +35,10 @@ end
 require_relative 'contrib'
 
 Datadog::DI::Contrib.load_now_or_later
+
+if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED'])
+  if Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
+    require_relative 'probe_file_loader'
+    Datadog::DI::ProbeFileLoader.load_now_or_later
+  end
+end

data/lib/datadog/di/code_tracker.rb

@@ -3,6 +3,7 @@
 # rubocop:disable Lint/AssignmentInCondition
 
 require_relative 'error'
+require_relative 'fatal_exceptions'
 
 module Datadog
   module DI
@@ -22,11 +23,115 @@ module Datadog
     class CodeTracker
       def initialize
         @registry = {}
+        @per_method_registry = {}
         @trace_point_lock = Mutex.new
         @registry_lock = Mutex.new
         @compiled_trace_point = nil
       end
 
+      # Populates the registry with iseqs for files that were loaded
+      # before code tracking started.
+      #
+      # Uses the all_iseqs C extension to walk the Ruby object space and
+      # find instruction sequences for already-loaded code. Whole-file
+      # iseqs are stored in the main registry; per-method/block/class
+      # iseqs are stored in per_method_registry as fallback for files
+      # whose whole-file iseq was GC'd.
+      #
+      # See docs/DynamicInstrumentationDevelopment.md "Iseq Lifecycle and GC"
+      # for which iseq types survive GC and implications for backfill.
+      #
+      # Whole-file detection uses two strategies:
+      # - Ruby 3.1+: DI.iseq_type (wraps rb_iseq_type) returns :top for
+      #   require/load and :main for the entry script. This is precise.
+      # - Ruby < 3.1: falls back to first_lineno == 0, which is true for
+      #   whole-file iseqs from require/load (INT2FIX(0) in Ruby's
+      #   rb_iseq_new_top and rb_iseq_new_main) and false for
+      #   method/block/class definitions (first_lineno >= 1).
+      #   InstructionSequence.compile passes first_lineno = 1 by default,
+      #   so eval'd code is not matched. Both strategies produce the same
+      #   result in practice.
+      #
+      # Does not overwrite iseqs already in the registry (from
+      # :script_compiled), since those are guaranteed to be whole-file
+      # iseqs and are authoritative.
+      #
+      # @return [void]
+      def backfill_registry
+        iseqs = DI.file_iseqs
+        have_iseq_type = DI.respond_to?(:iseq_type)
+        registry_lock.synchronize do
+          iseqs.each do |iseq|
+            path = iseq.absolute_path
+            next unless path
+
+            whole_file = if have_iseq_type
+              type = DI.iseq_type(iseq)
+              # Require first_lineno == 0 to exclude compile_file/compile
+              # iseqs. These are :top type but have first_lineno == 1 and
+              # produce iseq objects distinct from require-produced iseqs.
+              # Targeted TracePoints are bound to the specific iseq object
+              # — a probe on a compile_file iseq silently never fires when
+              # the require-produced code runs.
+              (type == :top || type == :main) && iseq.first_lineno == 0
+            else
+              iseq.first_lineno == 0
+            end
+
+            if whole_file
+              # Ruby 3.2.9+ creates dummy profiler iseqs during require/load
+              # (rb_iseq_alloc_with_dummy_path in iseq.c). These have type
+              # :top, first_lineno == 0, and the same absolute_path as the
+              # real iseq — but iseq_size == 0 (no bytecode). A targeted
+              # TracePoint on a dummy iseq can't find child iseqs and raises
+              # ArgumentError "can not enable any hooks". Filter them out:
+              # a real top-level iseq always has at least one trace event.
+              next if iseq.trace_points.empty?
+
+              # Do not overwrite entries from :script_compiled — those are
+              # captured at load time and are authoritative.
+              next if registry.key?(path)
+
+              registry[path] = iseq
+            else
+              # Skip top-level script iseqs (:top/:main) produced by
+              # RubyVM::InstructionSequence.compile_file and .compile
+              # (compile source to bytecode without executing it).
+              # These represent the file body,
+              # not a method or block. They pass the first_lineno check
+              # (lineno != 0) but a targeted TracePoint bound to one
+              # of these never fires for method-level code — the
+              # user's probe silently produces no snapshots.
+              #
+              # On Ruby < 3.1 (no iseq_type), we cannot distinguish
+              # these from method iseqs, so they leak into
+              # per_method_registry. If iseq_for_line selects a leaked
+              # top-level iseq instead of the real method iseq, the
+              # probe installs but silently never fires — same failure
+              # as above. This requires the application to call
+              # compile_file and hold the result, which is rare outside
+              # tooling like bootsnap (which discards it).
+              next if have_iseq_type && (type == :top || type == :main)
+
+              # Store per-method/block/class iseqs as fallback for files
+              # whose whole-file iseq was GC'd. These can be used to
+              # target line probes on lines within their range.
+              (per_method_registry[path] ||= []) << iseq
+            end
+          end
+        end
+        nil
+      rescue Exception => exc # standard:disable Lint/RescueException
+        Datadog::DI.reraise_if_fatal(exc)
+        # Backfill is best-effort — if it fails, line probes on
+        # pre-loaded code won't work but everything else is unaffected.
+        if component = DI.current_component
+          component.logger.debug { "di: backfill_registry failed: #{exc.class}: #{exc.message}" }
+          component.telemetry&.report(exc, description: "backfill_registry failed")
+        end
+        nil
+      end
+
       # Starts tracking loaded code.
       #
       # This method should generally be called early in application boot
@@ -88,13 +193,14 @@ module Datadog
           # Since this method normally is called from customer applications,
           # rescue any exceptions that might not be handled to not break said
           # customer applications.
-          rescue => exc
+          rescue Exception => exc # standard:disable Lint/RescueException
+            Datadog::DI.reraise_if_fatal(exc)
             # Code tracker may be loaded without the rest of DI,
             # in which case DI.component will not yet be defined,
             # but we will have DI.current_component (set to nil).
             if component = DI.current_component
               raise if component.settings.dynamic_instrumentation.internal.propagate_all_exceptions
-              component.logger.debug { "di: unhandled exception in script_compiled trace point: #{exc.class}: #{exc}" }
+              component.logger.debug { "di: unhandled exception in script_compiled trace point: #{exc.class}: #{exc.message}" }
               component.telemetry&.report(exc, description: "Unhandled exception in script_compiled trace point")
               # TODO test this path
             else
@@ -104,6 +210,12 @@ module Datadog
               # TODO test this path
             end
           end
+
+          # Backfill the registry with iseqs for files that were loaded
+          # before tracking started. This must happen after the trace
+          # point is enabled so that any files loaded concurrently are
+          # captured by the trace point (backfill won't overwrite them).
+          backfill_registry
         end
       end
 
@@ -140,23 +252,79 @@ module Datadog
           exact = registry[suffix]
           return [suffix, exact] if exact
 
-          suffix = suffix.dup
-          loop do
-            inexact = []
-            registry.each do |path, iseq|
-              if Utils.path_matches_suffix?(path, suffix)
-                inexact << [path, iseq]
+          # Normalize Windows-style backslash separators (DEBUG-5111) upfront
+          # so the suffix-shortening loop's "/+" regex can strip leading
+          # components on probes whose sourceFile uses backslashes.
+          suffix = Utils.normalize_windows_separators(suffix)
+
+          # Per the design comment in utils.rb, attempt case-sensitive
+          # matching first (steps 5-6) and only fall back to case-insensitive
+          # matching (steps 7-8) when no case-sensitive match is found.
+          [false, true].each do |case_insensitive|
+            working_suffix = suffix.dup
+            loop do
+              inexact = []
+              registry.each do |path, iseq|
+                if Utils.path_matches_suffix?(path, working_suffix, case_insensitive: case_insensitive)
+                  inexact << [path, iseq]
+                end
               end
+              if inexact.length > 1
+                raise Error::MultiplePathsMatch, "Multiple paths matched requested suffix"
+              end
+              if inexact.any?
+                return inexact.first
+              end
+              break unless working_suffix.include?('/')
+              working_suffix.sub!(%r{.*/+}, '')
             end
-            if inexact.length > 1
-              raise Error::MultiplePathsMatch, "Multiple paths matched requested suffix"
-            end
-            if inexact.any?
-              return inexact.first
+          end
+          nil
+        end
+      end
+
+      # Returns a [path, iseq] pair for a line probe target, or nil.
+      #
+      # First checks the whole-file iseq registry (via iseqs_for_path_suffix).
+      # If no whole-file iseq exists, searches the per-method iseq registry
+      # for an iseq whose trace_points include the target line.
+      #
+      # @param suffix [String] file path or suffix to match
+      # @param line [Integer] target line number
+      # @return [Array(String, RubyVM::InstructionSequence), nil]
+      def iseq_for_line(suffix, line)
+        # Try whole-file iseq first — it always covers all lines.
+        result = iseqs_for_path_suffix(suffix)
+        return result if result
+
+        # Fall back to per-method iseqs.
+        registry_lock.synchronize do
+          # Resolve the path using the per-method registry keys.
+          path = resolve_path_suffix(suffix, per_method_registry.keys)
+          return nil unless path
+
+          iseqs = per_method_registry[path]
+          return nil unless iseqs
+
+          # Only match event types the instrumenter subscribes to
+          # (:line, :return, :b_return — see hook_line). Lines that
+          # only carry :call (e.g. a `def` line within the defined
+          # method's own iseq, not the enclosing scope) have no
+          # subscribed event at that position; TracePoint#enable
+          # raises because it cannot bind an enabled event there.
+          matches = iseqs.select do |iseq|
+            iseq.trace_points.any? do |tp_line, event|
+              tp_line == line && (event == :line || event == :return || event == :b_return)
             end
-            return nil unless suffix.include?('/')
-            suffix.sub!(%r{.*/+}, '')
           end
+          # When multiple iseqs contain the target line (e.g. a method
+          # and an inline block sharing the same line), picking one
+          # would silently miss executions in the other context.
+          # Raise so the probe is recorded as failed with a clear error.
+          if matches.length > 1
+            raise Error::MultiplePathsMatch, "Multiple code locations match line #{line}"
+          end
+          matches.first ? [path, matches.first] : nil
         end
       end
 
@@ -186,6 +354,7 @@ module Datadog
       def clear
         registry_lock.synchronize do
           registry.clear
+          per_method_registry.clear
         end
       end
 
@@ -195,8 +364,41 @@ module Datadog
       # objects representing compiled code of those files.
       attr_reader :registry
 
+      # Mapping from paths to arrays of per-method/block/class iseqs.
+      # Used as fallback when the whole-file iseq has been GC'd.
+      attr_reader :per_method_registry
+
       attr_reader :trace_point_lock
       attr_reader :registry_lock
+
+      # Resolves a path suffix against a set of known paths.
+      # Returns the matching path or nil.
+      #
+      # Must be called within registry_lock.
+      def resolve_path_suffix(suffix, paths)
+        # Exact match.
+        return suffix if paths.include?(suffix)
+
+        # Normalize Windows-style backslash separators (DEBUG-5111) upfront
+        # so the suffix-shortening loop's "/+" regex can strip leading
+        # components on probes whose sourceFile uses backslashes.
+        suffix = Utils.normalize_windows_separators(suffix)
+
+        # Suffix match. Per the design comment in utils.rb, attempt
+        # case-sensitive matching first (steps 5-6) and only fall back to
+        # case-insensitive (steps 7-8) when no case-sensitive match is found.
+        [false, true].each do |case_insensitive|
+          working_suffix = suffix.dup
+          loop do
+            matches = paths.select { |p| Utils.path_matches_suffix?(p, working_suffix, case_insensitive: case_insensitive) }
+            raise Error::MultiplePathsMatch, "Multiple paths matched requested suffix" if matches.length > 1
+            return matches.first if matches.any?
+            break unless working_suffix.include?('/')
+            working_suffix.sub!(%r{.*/+}, '')
+          end
+        end
+        nil
+      end
     end
   end
 end