datadog 2.8.0 → 2.39.0

data/lib/datadog/data_streams/transport/http/stats.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative "../stats"
+require_relative "../../../core/transport/http/api/endpoint"
+require_relative "../../../core/transport/http/response"
+
+module Datadog
+  module DataStreams
+    module Transport
+      module HTTP
+        # HTTP transport behavior for Data Streams stats
+        module Stats
+          # Response from HTTP transport for DSM stats
+          class Response
+            include Datadog::Core::Transport::HTTP::Response
+
+            def initialize(http_response)
+              super
+            end
+          end
+
+          module API
+            # Endpoint for submitting DSM stats data
+            class Endpoint < Core::Transport::HTTP::API::Endpoint
+              def initialize(path)
+                super(:post, path)
+              end
+
+              def call(env, &block)
+                # Build request
+                env.verb = verb
+                env.path = path
+                env.body = env.request.parcel.data
+                if (content_type = env.request.parcel.content_type)
+                  env.headers["content-type"] = content_type
+                end
+                if (content_encoding = env.request.parcel.content_encoding)
+                  env.headers["content-encoding"] = content_encoding
+                end
+
+                # Send request
+                http_response = yield(env)
+
+                # Build response
+                Response.new(http_response)
+              end
+
+              def encoder
+                # DSM handles encoding in the transport layer
+                nil
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams/transport/http.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "../../core/transport/http"
+require_relative "http/stats"
+require_relative "stats"
+
+module Datadog
+  module DataStreams
+    module Transport
+      # HTTP transport for Data Streams Monitoring
+      module HTTP
+        V01 = Stats::API::Endpoint.new(
+          "/v0.1/pipeline_stats"
+        )
+
+        module_function
+
+        # Builds a new Transport::HTTP::Client with default settings
+        def default(
+          agent_settings:,
+          logger:
+        )
+          Core::Transport::HTTP.build(
+            agent_settings: agent_settings,
+            logger: logger,
+          ) do |transport|
+            transport.api "v0.1", V01, default: true
+
+            # Call block to apply any customization, if provided
+            yield(transport) if block_given?
+          end.to_transport(Transport::Stats::Transport)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams/transport/stats.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "msgpack"
+require "zlib"
+require_relative "../../core/transport/parcel"
+require_relative "../../core/transport/request"
+require_relative "../../core/transport/transport"
+
+module Datadog
+  module DataStreams
+    module Transport
+      module Stats
+        # Request for DSM stats
+        class Request < Datadog::Core::Transport::Request
+        end
+
+        # Transport for Data Streams Monitoring stats
+        class Transport < Core::Transport::Transport
+          def send_stats(payload)
+            # MessagePack encode and gzip compress the payload
+            msgpack_data = MessagePack.pack(payload)
+            compressed_data = Zlib.gzip(msgpack_data)
+
+            # Create parcel and request
+            parcel = Core::Transport::Parcel.new(
+              compressed_data,
+              content_type: "application/msgpack",
+              content_encoding: "gzip",
+            )
+            request = Request.new(parcel)
+
+            # Send to agent
+            client.send_request(:stats, request)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require_relative "data_streams/processor"
+require_relative "data_streams/pathway_context"
+require_relative "data_streams/configuration"
+require_relative "data_streams/extensions"
+require_relative "core/utils/time"
+
+module Datadog
+  # Datadog Data Streams Monitoring public API.
+  #
+  # The Datadog team ensures that public methods in this module
+  # only receive backwards compatible changes, and breaking changes
+  # will only occur in new major versions releases.
+  # @public_api
+  module DataStreams
+    class << self
+      # Set a produce checkpoint for Data Streams Monitoring
+      #
+      # @param type [String] The type of the checkpoint (e.g., 'kafka', 'kinesis', 'sns')
+      # @param destination [String] The destination (e.g., topic, exchange, stream name)
+      # @param auto_instrumentation [Boolean] Whether this checkpoint was set by auto-instrumentation (default: false)
+      # @param tags [Hash] Additional tags to include
+      # @yield [key, value] Block to inject context into carrier
+      # @return [String, nil] Base64 encoded pathway context or nil if disabled
+      # @public_api
+      def set_produce_checkpoint(type:, destination:, auto_instrumentation: false, tags: {}, &block)
+        processor&.set_produce_checkpoint(
+          type: type,
+          destination: destination,
+          manual_checkpoint: !auto_instrumentation,
+          tags: tags,
+          &block
+        )
+      end
+
+      # Set a consume checkpoint for Data Streams Monitoring
+      #
+      # @param type [String] The type of the checkpoint (e.g., 'kafka', 'kinesis', 'sns')
+      # @param source [String] The source (e.g., topic, exchange, stream name)
+      # @param auto_instrumentation [Boolean] Whether this checkpoint was set by auto-instrumentation (default: false)
+      # @param tags [Hash] Additional tags to include
+      # @yield [key] Block to extract context from carrier
+      # @return [String, nil] Base64 encoded pathway context or nil if disabled
+      # @public_api
+      def set_consume_checkpoint(type:, source:, auto_instrumentation: false, tags: {}, &block)
+        processor&.set_consume_checkpoint(
+          type: type,
+          source: source,
+          manual_checkpoint: !auto_instrumentation,
+          tags: tags,
+          &block
+        )
+      end
+
+      # Track Kafka produce offset for lag monitoring
+      #
+      # @param topic [String] The Kafka topic name
+      # @param partition [Integer] The partition number
+      # @param offset [Integer] The offset of the produced message
+      # @return [Boolean, nil] true if tracking succeeded, nil if disabled
+      # @!visibility private
+      def track_kafka_produce(topic, partition, offset)
+        processor&.track_kafka_produce(topic, partition, offset, Core::Utils::Time.now)
+      end
+
+      # Track Kafka message consumption for consumer lag monitoring
+      #
+      # @param topic [String] The Kafka topic name
+      # @param partition [Integer] The partition number
+      # @param offset [Integer] The offset of the consumed message
+      # @return [Boolean, nil] true if tracking succeeded, nil if disabled
+      # @!visibility private
+      def track_kafka_consume(topic, partition, offset)
+        processor&.track_kafka_consume(topic, partition, offset, Core::Utils::Time.now)
+      end
+
+      # Check if Data Streams Monitoring is enabled and available
+      #
+      # @return [Boolean] true if the processor is available
+      # @public_api
+      def enabled?
+        !processor.nil?
+      end
+
+      private
+
+      def processor
+        components.data_streams
+      end
+
+      def components
+        Datadog.send(:components)
+      end
+    end
+
+    # Expose Data Streams to global shared objects
+    Extensions.activate!
+  end
+end

data/lib/datadog/di/base.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+# This file is loaded by datadog/di/preload.rb.
+# It contains just the global DI reference to the (normally one and only)
+# code tracker for the current process.
+# This file should not require the rest of DI, specifically none of the
+# contrib code that is meant to be loaded after third-party libraries
+# are loaded, and also none of the rest of datadog library which also
+# has contrib code in other products.
+
+require_relative "fatal_exceptions"
+require_relative "code_tracker"
+
+# Needed since this file can be loaded without core
+require_relative "../ruby_version"
+
+module Datadog
+  # Namespace for Datadog dynamic instrumentation.
+  #
+  # @api private
+  module DI
+    LOCK = Mutex.new
+
+    # Initialized eagerly to avoid "instance variable not initialized"
+    # warning on Ruby 2.6/2.7 and to simplify the type to non-nullable.
+    @current_components = []
+
+    class << self
+      attr_reader :code_tracker
+
+      # Activates code tracking. Normally this method should be called
+      # when the application starts. If instrumenting third-party code,
+      # code tracking needs to be enabled before the third-party libraries
+      # are loaded. Any third-party code loaded before code tracking is
+      # activated will NOT be instrumentable using dynamic instrumentation.
+      #
+      # TODO test that activating tracker multiple times preserves
+      # existing mappings in the registry
+      def activate_tracking!
+        (@code_tracker ||= CodeTracker.new).start
+      end
+
+      # Activates code tracking if possible.
+      #
+      # This method does nothing if invoked in an environment that does not
+      # implement required trace points for code tracking (MRI Ruby < 2.6,
+      # JRuby) and rescues any exceptions that may be raised by downstream
+      # DI code.
+      def activate_tracking
+        # :script_compiled trace point was added in Ruby 2.6.
+        return unless RubyVersion.is?(">= 2.6")
+
+        begin
+          # Activate code tracking by default because line trace points will not work
+          # without it.
+          Datadog::DI.activate_tracking!
+        rescue Exception => exc # standard:disable Lint/RescueException
+          Datadog::DI.reraise_if_fatal(exc)
+          if defined?(Datadog.logger)
+            Datadog.logger.warn { "di: Failed to activate code tracking for DI: #{exc.class}: #{exc.message}" }
+          else
+            # We do not have Datadog logger potentially because DI code tracker is
+            # being loaded early in application boot process and the rest of datadog
+            # wasn't loaded yet. Output to standard error.
+            warn("datadog: di: Failed to activate code tracking for DI: #{exc.class}: #{exc.message}")
+          end
+        end
+      end
+
+      # Deactivates code tracking. In normal usage of DI this method should
+      # never be called, however it is used by DI's test suite to reset
+      # state for individual tests.
+      #
+      # Note that deactivating tracking clears out the registry, losing
+      # the ability to look up files that have been loaded into the process
+      # already.
+      def deactivate_tracking!
+        code_tracker&.stop
+      end
+
+      # Returns whether code tracking is available.
+      # This method should be used instead of querying #code_tracker
+      # because the latter one may be nil.
+      def code_tracking_active?
+        code_tracker&.active? || false
+      end
+
+      # DI code tracker is instantiated globally before the regular set of
+      # components is created, but the code tracker needs to call out to the
+      # "current" DI component to perform instrumentation when application
+      # code is loaded. Because this call may happen prior to Datadog
+      # components having been initialized, we maintain the "current component"
+      # which contains a reference to the most recently instantiated
+      # DI::Component. This way, if a DI component hasn't been instantiated,
+      # we do not try to reference Datadog.components.
+      # In other words, this method exists so that we never attempt to call
+      # Datadog.components from the code tracker.
+      def current_component
+        LOCK.synchronize do
+          @current_components.last
+        end
+      end
+
+      # To avoid potential races with DI::Component being added and removed,
+      # we maintain a list of the components. Normally the list should contain
+      # either zero or one component depending on whether DI is enabled in
+      # Datadog configuration. However, if a new instance of DI::Component
+      # is created while the previous instance is still running, we are
+      # guaranteed to not end up with no component when one is running.
+      def add_current_component(component)
+        LOCK.synchronize do
+          @current_components << component
+        end
+      end
+
+      def remove_current_component(component)
+        LOCK.synchronize do
+          @current_components.delete(component)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/boot.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative "logger"
+require_relative "base"
+require_relative "error"
+require_relative "code_tracker"
+require_relative "component"
+require_relative "context"
+require_relative "capture_limits"
+require_relative "capture_expression"
+require_relative "capture_expression_evaluator"
+require_relative "instrumenter"
+require_relative "probe"
+require_relative "probe_builder"
+require_relative "probe_manager"
+require_relative "probe_notification_builder"
+require_relative "probe_notifier_worker"
+require_relative "probe_repository"
+require_relative "redactor"
+require_relative "serializer"
+require_relative "transport/http"
+require_relative "utils"
+
+if %w[1 true yes].include?(Datadog::DATADOG_ENV["DD_DYNAMIC_INSTRUMENTATION_ENABLED"])
+
+  # For initial release of Dynamic Instrumentation, activate code tracking
+  # only if DI is explicitly requested in the environment.
+  # Code tracking is required for line probes to work; see the comments
+  # above for the implementation of the method.
+  #
+  # If DI is enabled programmatically, the application can (and must,
+  # for line probes to work) activate tracking in an initializer.
+  # We seem to have Datadog.logger here already
+  Datadog.logger.debug("di: activating code tracking")
+  Datadog::DI.activate_tracking
+end
+
+require_relative "contrib"
+
+Datadog::DI::Contrib.load_now_or_later
+
+if %w[1 true yes].include?(Datadog::DATADOG_ENV["DD_DYNAMIC_INSTRUMENTATION_ENABLED"])
+  if Datadog::DATADOG_ENV["DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE"]
+    require_relative "probe_file_loader"
+    Datadog::DI::ProbeFileLoader.load_now_or_later
+  end
+end

data/lib/datadog/di/capture_expression.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative "capture_limits"
+
+module Datadog
+  module DI
+    class CaptureExpression
+      def initialize(name:, expr:, limits: nil)
+        @name = name
+        @expr = expr
+        @limits = limits
+      end
+
+      attr_reader :name
+
+      attr_reader :expr
+
+      attr_reader :limits
+    end
+  end
+end

data/lib/datadog/di/capture_expression_evaluator.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require_relative "capture_expression"
+require_relative "capture_limits"
+require_relative "fatal_exceptions"
+
+module Datadog
+  module DI
+    class CaptureExpressionEvaluator
+      TELEMETRY_NAMESPACE = "dynamic_instrumentation"
+
+      def initialize(settings:, serializer:, logger:, telemetry: nil)
+        @settings = settings
+        @serializer = serializer
+        @logger = logger
+        @telemetry = telemetry
+      end
+
+      attr_reader :settings
+
+      attr_reader :serializer
+
+      attr_reader :logger
+
+      attr_reader :telemetry
+
+      def evaluate(probe, context)
+        budget_ns = settings.dynamic_instrumentation.max_time_to_serialize_ms * 1_000_000
+        deadline_ns = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :nanosecond) + budget_ns
+
+        output = {}
+        evaluation_errors = []
+
+        probe.capture_expressions.each do |capture_expression|
+          name = capture_expression.name
+
+          if ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :nanosecond) >= deadline_ns
+            output[name] = {notCapturedReason: "timeout"}
+            telemetry&.inc(TELEMETRY_NAMESPACE, "capture_expressions_skipped_by_timeout", 1)
+            next
+          end
+
+          begin
+            value = capture_expression.expr.evaluate(context)
+            limits = CaptureLimits.resolve(
+              expr_limits: capture_expression.limits,
+              probe: probe,
+              settings: settings,
+            )
+            output[name] = serializer.serialize_value(
+              value, name: name,
+              depth: limits[:depth],
+              attribute_count: limits[:attribute_count],
+              length: limits[:length],
+              collection_size: limits[:collection_size],
+            )
+          rescue Exception => exc # standard:disable Lint/RescueException
+            Datadog::DI.reraise_if_fatal(exc)
+            evaluation_errors << {expr: name, message: "#{exc.class}: #{exc.message}"}
+            logger.debug do
+              "di: probe #{probe.id}: capture expression #{name}: evaluation failed: #{exc.class}: #{exc.message}"
+            end
+            telemetry&.report(exc, description: "DI capture-expression evaluation failed")
+          end
+        end
+
+        [output, evaluation_errors]
+      end
+    end
+  end
+end

data/lib/datadog/di/capture_limits.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    class CaptureLimits
+      def initialize(max_reference_depth: nil, max_collection_size: nil,
+        max_length: nil, max_field_count: nil)
+        @max_reference_depth = max_reference_depth
+        @max_collection_size = max_collection_size
+        @max_length = max_length
+        @max_field_count = max_field_count
+      end
+
+      attr_reader :max_reference_depth
+
+      attr_reader :max_collection_size
+
+      attr_reader :max_length
+
+      attr_reader :max_field_count
+
+      def self.resolve(expr_limits:, probe:, settings:)
+        di = settings.dynamic_instrumentation
+        {
+          depth: expr_limits&.max_reference_depth ||
+            probe.max_capture_depth ||
+            di.max_capture_depth,
+          collection_size: expr_limits&.max_collection_size ||
+            probe.max_capture_collection_size ||
+            di.max_capture_collection_size,
+          length: expr_limits&.max_length ||
+            probe.max_capture_string_length ||
+            di.max_capture_string_length,
+          attribute_count: expr_limits&.max_field_count ||
+            probe.max_capture_attribute_count ||
+            di.max_capture_attribute_count,
+        }
+      end
+    end
+  end
+end