cw-datadog 2.23.0.2

data/lib/datadog/data_streams/transport/http/client.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative '../../../core/transport/http/response'
+
+module Datadog
+  module DataStreams
+    module Transport
+      module HTTP
+        # HTTP client for Data Streams Monitoring
+        class Client
+          attr_reader :api, :logger
+
+          def initialize(api, logger:)
+            @api = api
+            @logger = logger
+          end
+
+          def send_stats_payload(request)
+            send_request(request) do |api, env|
+              api.send_stats(env)
+            end
+          end
+
+          private
+
+          def send_request(request, &block)
+            # Build request into env
+            env = build_env(request)
+
+            # Get response from API
+            yield(api, env)
+          rescue => e
+            message =
+              "Internal error during #{self.class.name} request. Cause: #{e.class}: #{e} " \
+                "Location: #{Array(e.backtrace).first}"
+
+            logger.debug(message)
+
+            Datadog::Core::Transport::InternalErrorResponse.new(e)
+          end
+
+          def build_env(request)
+            Datadog::Core::Transport::HTTP::Env.new(request)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams/transport/http/stats.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require_relative '../stats'
+require_relative 'client'
+require_relative '../../../core/transport/http/response'
+require_relative '../../../core/transport/http/api/endpoint'
+require_relative '../../../core/transport/http/api/spec'
+require_relative '../../../core/transport/http/api/instance'
+
+module Datadog
+  module DataStreams
+    module Transport
+      module HTTP
+        # HTTP transport behavior for Data Streams stats
+        module Stats
+          # Response from HTTP transport for DSM stats
+          class Response
+            include Datadog::Core::Transport::HTTP::Response
+
+            def initialize(http_response)
+              super
+            end
+          end
+
+          module API
+            # HTTP API Spec for DSM
+            class Spec < Core::Transport::HTTP::API::Spec
+              attr_accessor :stats
+
+              def send_stats(env, &block)
+                raise Core::Transport::HTTP::API::Spec::EndpointNotDefinedError.new('stats', self) if stats.nil?
+
+                stats.call(env, &block)
+              end
+
+              def encoder
+                # DSM handles encoding in the transport layer (MessagePack + gzip)
+                # so we don't need an encoder at the API level
+                nil
+              end
+            end
+
+            # HTTP API Instance for DSM
+            class Instance < Core::Transport::HTTP::API::Instance
+              def send_stats(env)
+                unless spec.is_a?(Stats::API::Spec)
+                  raise Core::Transport::HTTP::API::Instance::EndpointNotSupportedError.new(
+                    'stats', self
+                  )
+                end
+
+                spec.send_stats(env) do |request_env|
+                  call(request_env)
+                end
+              end
+            end
+
+            # Endpoint for submitting DSM stats data
+            class Endpoint < Core::Transport::HTTP::API::Endpoint
+              def initialize(path)
+                super(:post, path)
+              end
+
+              def call(env, &block)
+                # Build request
+                env.verb = verb
+                env.path = path
+                env.body = env.request.parcel.data
+
+                # Send request
+                http_response = yield(env)
+
+                # Build response
+                Response.new(http_response)
+              end
+
+              def encoder
+                # DSM handles encoding in the transport layer
+                nil
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams/transport/http.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative '../../core/transport/http'
+require_relative 'http/api'
+require_relative 'http/client'
+require_relative 'http/stats'
+require_relative 'stats'
+
+module Datadog
+  module DataStreams
+    module Transport
+      # HTTP transport for Data Streams Monitoring
+      module HTTP
+        module_function
+
+        # Builds a new Transport::HTTP::Client with default settings
+        def default(
+          agent_settings:,
+          logger:
+        )
+          Core::Transport::HTTP.build(
+            api_instance_class: Stats::API::Instance,
+            agent_settings: agent_settings,
+            logger: logger,
+            headers: {
+              'Content-Type' => 'application/msgpack',
+              'Content-Encoding' => 'gzip'
+            }
+          ) do |transport|
+            apis = API.defaults
+
+            transport.api API::V01, apis[API::V01], default: true
+
+            # Call block to apply any customization, if provided
+            yield(transport) if block_given?
+          end.to_transport(Transport::Stats::Transport)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams/transport/stats.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'msgpack'
+require 'zlib'
+require_relative '../../core/transport/parcel'
+require_relative '../../core/transport/request'
+
+module Datadog
+  module DataStreams
+    module Transport
+      module Stats
+        # Parcel for encoded DSM stats payload
+        class EncodedParcel
+          include Datadog::Core::Transport::Parcel
+
+          def initialize(data)
+            @data = data
+          end
+
+          attr_reader :data
+        end
+
+        # Request for DSM stats
+        class Request < Datadog::Core::Transport::Request
+        end
+
+        # Transport for Data Streams Monitoring stats
+        class Transport
+          attr_reader :client, :apis, :current_api_id, :logger
+
+          def initialize(apis, default_api, logger:)
+            @apis = apis
+            @logger = logger
+            @default_api = default_api
+            @current_api_id = default_api
+
+            @client = HTTP::Client.new(current_api, logger: @logger)
+          end
+
+          def send_stats(payload)
+            # MessagePack encode and gzip compress the payload
+            msgpack_data = MessagePack.pack(payload)
+            compressed_data = Zlib.gzip(msgpack_data)
+
+            # Create parcel and request
+            parcel = EncodedParcel.new(compressed_data)
+            request = Request.new(parcel)
+
+            # Send to agent
+            client.send_stats_payload(request)
+          end
+
+          def current_api
+            apis[@current_api_id]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/data_streams.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require_relative 'data_streams/processor'
+require_relative 'data_streams/pathway_context'
+require_relative 'data_streams/configuration/settings'
+require_relative 'data_streams/extensions'
+require_relative 'core/utils/time'
+
+module Datadog
+  # Datadog Data Streams Monitoring public API.
+  #
+  # The Datadog team ensures that public methods in this module
+  # only receive backwards compatible changes, and breaking changes
+  # will only occur in new major versions releases.
+  # @public_api
+  module DataStreams
+    class << self
+      # Set a produce checkpoint for Data Streams Monitoring
+      #
+      # @param type [String] The type of the checkpoint (e.g., 'kafka', 'kinesis', 'sns')
+      # @param destination [String] The destination (e.g., topic, exchange, stream name)
+      # @param auto_instrumentation [Boolean] Whether this checkpoint was set by auto-instrumentation (default: false)
+      # @param tags [Hash] Additional tags to include
+      # @yield [key, value] Block to inject context into carrier
+      # @return [String, nil] Base64 encoded pathway context or nil if disabled
+      # @public_api
+      def set_produce_checkpoint(type:, destination:, auto_instrumentation: false, tags: {}, &block)
+        processor&.set_produce_checkpoint(
+          type: type,
+          destination: destination,
+          manual_checkpoint: !auto_instrumentation,
+          tags: tags,
+          &block
+        )
+      end
+
+      # Set a consume checkpoint for Data Streams Monitoring
+      #
+      # @param type [String] The type of the checkpoint (e.g., 'kafka', 'kinesis', 'sns')
+      # @param source [String] The source (e.g., topic, exchange, stream name)
+      # @param auto_instrumentation [Boolean] Whether this checkpoint was set by auto-instrumentation (default: false)
+      # @param tags [Hash] Additional tags to include
+      # @yield [key] Block to extract context from carrier
+      # @return [String, nil] Base64 encoded pathway context or nil if disabled
+      # @public_api
+      def set_consume_checkpoint(type:, source:, auto_instrumentation: false, tags: {}, &block)
+        processor&.set_consume_checkpoint(
+          type: type,
+          source: source,
+          manual_checkpoint: !auto_instrumentation,
+          tags: tags,
+          &block
+        )
+      end
+
+      # Track Kafka produce offset for lag monitoring
+      #
+      # @param topic [String] The Kafka topic name
+      # @param partition [Integer] The partition number
+      # @param offset [Integer] The offset of the produced message
+      # @return [Boolean, nil] true if tracking succeeded, nil if disabled
+      # @!visibility private
+      def track_kafka_produce(topic, partition, offset)
+        processor&.track_kafka_produce(topic, partition, offset, Core::Utils::Time.now)
+      end
+
+      # Track Kafka message consumption for consumer lag monitoring
+      #
+      # @param topic [String] The Kafka topic name
+      # @param partition [Integer] The partition number
+      # @param offset [Integer] The offset of the consumed message
+      # @return [Boolean, nil] true if tracking succeeded, nil if disabled
+      # @!visibility private
+      def track_kafka_consume(topic, partition, offset)
+        processor&.track_kafka_consume(topic, partition, offset, Core::Utils::Time.now)
+      end
+
+      # Check if Data Streams Monitoring is enabled and available
+      #
+      # @return [Boolean] true if the processor is available
+      # @public_api
+      def enabled?
+        !processor.nil?
+      end
+
+      private
+
+      def processor
+        components.data_streams
+      end
+
+      def components
+        Datadog.send(:components)
+      end
+    end
+
+    # Expose Data Streams to global shared objects
+    Extensions.activate!
+  end
+end

data/lib/datadog/di/base.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+# This file is loaded by datadog/di/preload.rb.
+# It contains just the global DI reference to the (normally one and only)
+# code tracker for the current process.
+# This file should not require the rest of DI, specifically none of the
+# contrib code that is meant to be loaded after third-party libraries
+# are loaded, and also none of the rest of datadog library which also
+# has contrib code in other products.
+
+require_relative 'code_tracker'
+
+module Datadog
+  # Namespace for Datadog dynamic instrumentation.
+  #
+  # @api private
+  module DI
+    LOCK = Mutex.new
+
+    class << self
+      attr_reader :code_tracker
+
+      # Activates code tracking. Normally this method should be called
+      # when the application starts. If instrumenting third-party code,
+      # code tracking needs to be enabled before the third-party libraries
+      # are loaded. Any third-party code loaded before code tracking is
+      # activated will NOT be instrumentable using dynamic instrumentation.
+      #
+      # TODO test that activating tracker multiple times preserves
+      # existing mappings in the registry
+      def activate_tracking!
+        (@code_tracker ||= CodeTracker.new).start
+      end
+
+      # Activates code tracking if possible.
+      #
+      # This method does nothing if invoked in an environment that does not
+      # implement required trace points for code tracking (MRI Ruby < 2.6,
+      # JRuby) and rescues any exceptions that may be raised by downstream
+      # DI code.
+      def activate_tracking
+        # :script_compiled trace point was added in Ruby 2.6.
+        return unless RUBY_VERSION >= '2.6'
+
+        begin
+          # Activate code tracking by default because line trace points will not work
+          # without it.
+          Datadog::DI.activate_tracking!
+        rescue => exc
+          if defined?(Datadog.logger)
+            Datadog.logger.warn { "di: Failed to activate code tracking for DI: #{exc.class}: #{exc}" }
+          else
+            # We do not have Datadog logger potentially because DI code tracker is
+            # being loaded early in application boot process and the rest of datadog
+            # wasn't loaded yet. Output to standard error.
+            warn("datadog: di: Failed to activate code tracking for DI: #{exc.class}: #{exc}")
+          end
+        end
+      end
+
+      # Deactivates code tracking. In normal usage of DI this method should
+      # never be called, however it is used by DI's test suite to reset
+      # state for individual tests.
+      #
+      # Note that deactivating tracking clears out the registry, losing
+      # the ability to look up files that have been loaded into the process
+      # already.
+      def deactivate_tracking!
+        code_tracker&.stop
+      end
+
+      # Returns whether code tracking is available.
+      # This method should be used instead of querying #code_tracker
+      # because the latter one may be nil.
+      def code_tracking_active?
+        code_tracker&.active? || false
+      end
+
+      # DI code tracker is instantiated globally before the regular set of
+      # components is created, but the code tracker needs to call out to the
+      # "current" DI component to perform instrumentation when application
+      # code is loaded. Because this call may happen prior to Datadog
+      # components having been initialized, we maintain the "current component"
+      # which contains a reference to the most recently instantiated
+      # DI::Component. This way, if a DI component hasn't been instantiated,
+      # we do not try to reference Datadog.components.
+      # In other words, this method exists so that we never attempt to call
+      # Datadog.components from the code tracker.
+      def current_component
+        LOCK.synchronize do
+          @current_components&.last
+        end
+      end
+
+      # To avoid potential races with DI::Component being added and removed,
+      # we maintain a list of the components. Normally the list should contain
+      # either zero or one component depending on whether DI is enabled in
+      # Datadog configuration. However, if a new instance of DI::Component
+      # is created while the previous instance is still running, we are
+      # guaranteed to not end up with no component when one is running.
+      def add_current_component(component)
+        LOCK.synchronize do
+          @current_components ||= []
+          @current_components << component
+        end
+      end
+
+      def remove_current_component(component)
+        LOCK.synchronize do
+          @current_components&.delete(component)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/di/boot.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative 'logger'
+require_relative 'base'
+require_relative 'error'
+require_relative 'code_tracker'
+require_relative 'component'
+require_relative 'context'
+require_relative 'instrumenter'
+require_relative 'probe'
+require_relative 'probe_builder'
+require_relative 'probe_manager'
+require_relative 'probe_notification_builder'
+require_relative 'probe_notifier_worker'
+require_relative 'redactor'
+require_relative 'serializer'
+require_relative 'transport/http'
+require_relative 'utils'
+
+if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore
+
+  # For initial release of Dynamic Instrumentation, activate code tracking
+  # only if DI is explicitly requested in the environment.
+  # Code tracking is required for line probes to work; see the comments
+  # above for the implementation of the method.
+  #
+  # If DI is enabled programmatically, the application can (and must,
+  # for line probes to work) activate tracking in an initializer.
+  # We seem to have Datadog.logger here already
+  Datadog.logger.debug("di: activating code tracking")
+  Datadog::DI.activate_tracking
+end
+
+require_relative 'contrib'
+
+Datadog::DI::Contrib.load_now_or_later
+
+if %w[1 true yes].include?(Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_ENABLED']) # steep:ignore
+  if Datadog::DATADOG_ENV['DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE']
+    require_relative 'probe_file_loader'
+    Datadog::DI::ProbeFileLoader.load_now_or_later
+  end
+end