datadog 2.22.0 → 2.24.0

data/lib/datadog/di/transport/http.rb

@@ -1,31 +1,46 @@
 # frozen_string_literal: true
 
+require_relative '../../core/encoding'
+require_relative '../../core/transport/http'
 require_relative 'diagnostics'
 require_relative 'input'
-require_relative 'http/api'
-require_relative '../../core/transport/http'
 
 module Datadog
   module DI
     module Transport
       # Namespace for HTTP transport components
       module HTTP
-        module_function
+        DIAGNOSTICS = Diagnostics::API::Endpoint.new(
+          '/debugger/v1/diagnostics',
+          Core::Encoding::JSONEncoder,
+        )
+
+        INPUT = Input::API::Endpoint.new(
+          '/debugger/v2/input',
+          Core::Encoding::JSONEncoder,
+        )
+
+        LEGACY_INPUT = Input::API::Endpoint.new(
+          # We used to use /debugger/v1/input, but now input
+          # payloads should be going to the diagnostics endpoint
+          # which I gather performs data redaction.
+          '/debugger/v1/diagnostics',
+          Core::Encoding::JSONEncoder,
+        )
 
         # Builds a new Transport::HTTP::Client with default settings
         # Pass a block to override any settings.
-        def diagnostics(
+        def self.diagnostics(
           agent_settings:,
           logger:,
-          api_version: nil,
           headers: nil
         )
-          Core::Transport::HTTP.build(api_instance_class: Diagnostics::API::Instance,
+          Core::Transport::HTTP.build(
             logger: logger,
-            agent_settings: agent_settings, api_version: api_version, headers: headers) do |transport|
-            apis = API.defaults
-
-            transport.api API::DIAGNOSTICS, apis[API::DIAGNOSTICS]
+            agent_settings: agent_settings,
+            headers: headers,
+          ) do |transport|
+            transport.api 'diagnostics', DIAGNOSTICS
 
             # Call block to apply any customization, if provided
             yield(transport) if block_given?
@@ -34,18 +49,18 @@ module Datadog
 
         # Builds a new Transport::HTTP::Client with default settings
         # Pass a block to override any settings.
-        def input(
+        def self.input(
           agent_settings:,
           logger:,
-          api_version: nil,
           headers: nil
         )
-          Core::Transport::HTTP.build(api_instance_class: Input::API::Instance,
+          Core::Transport::HTTP.build(
             logger: logger,
-            agent_settings: agent_settings, api_version: api_version, headers: headers) do |transport|
-            apis = API.defaults
-
-            transport.api API::INPUT, apis[API::INPUT]
+            agent_settings: agent_settings,
+            headers: headers,
+          ) do |transport|
+            transport.api 'input', INPUT, fallback: 'legacy_input', default: true
+            transport.api 'legacy_input', LEGACY_INPUT
 
             # Call block to apply any customization, if provided
             yield(transport) if block_given?

data/lib/datadog/di/transport/input.rb

@@ -1,7 +1,13 @@
 # frozen_string_literal: true
 
+require_relative '../../core/chunker'
+require_relative '../../core/encoding'
+require_relative '../../core/tag_builder'
 require_relative '../../core/transport/parcel'
-require_relative 'http/client'
+require_relative '../../core/transport/request'
+require_relative '../../core/transport/transport'
+require_relative '../error'
+require_relative 'http/input'
 
 module Datadog
   module DI
@@ -21,47 +27,74 @@ module Datadog
           end
         end
 
-        class Transport
-          attr_reader :client, :apis, :default_api, :current_api_id, :logger
+        class Transport < Core::Transport::Transport
+          # The limit on an individual snapshot payload, aka "log line",
+          # is 1 MB.
+          #
+          # TODO There is an RFC for snapshot pruning that should be
+          # implemented to reduce the size of snapshots to be below this
+          # limit, so that we can send a portion of the captured data
+          # rather than dropping the snapshot entirely.
+          MAX_SERIALIZED_SNAPSHOT_SIZE = 1024 * 1024
 
-          def initialize(apis, default_api, logger:)
-            @apis = apis
-            @logger = logger
+          # The maximum chunk (batch) size that intake permits is 5 MB.
+          #
+          # Two bytes are for the [ and ] of JSON array syntax.
+          MAX_CHUNK_SIZE = 5 * 1024 * 1024 - 2
 
-            @client = HTTP::Client.new(current_api, logger: logger)
-          end
-
-          def current_api
-            @apis[HTTP::API::INPUT]
-          end
+          # Try to send smaller payloads to avoid large network requests.
+          # If a payload is larger than default chunk size but is under the
+          # max chunk size, it will still get sent out.
+          DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024
 
           def send_input(payload, tags)
-            json = JSON.dump(payload)
-            parcel = EncodedParcel.new(json)
+            # Tags are the same for all chunks, serialize them one time.
             serialized_tags = Core::TagBuilder.serialize_tags(tags)
+
+            encoder = Core::Encoding::JSONEncoder
+            encoded_snapshots = Core::Utils::Array.filter_map(payload) do |snapshot|
+              encoded = encoder.encode(snapshot)
+              if encoded.length > MAX_SERIALIZED_SNAPSHOT_SIZE
+                # Drop the snapshot.
+                # TODO report via telemetry metric?
+                logger.debug { "di: dropping too big snapshot" }
+                nil
+              else
+                encoded
+              end
+            end
+
+            Datadog::Core::Chunker.chunk_by_size(
+              encoded_snapshots, DEFAULT_CHUNK_SIZE,
+            ).each do |chunk|
+              # We drop snapshots that are too big earlier.
+              # The limit on chunked payload length here is greater
+              # than the limit on snapshot size, therefore no chunks
+              # can exceed limits here.
+              chunked_payload = encoder.join(chunk)
+
+              # We need to rescue exceptions for each chunk so that
+              # subsequent chunks are attempted to be sent.
+              begin
+                send_input_chunk(chunked_payload, serialized_tags)
+              rescue => exc
+                logger.debug { "di: failed to send snapshot chunk: #{exc.class}: #{exc} (at #{exc.backtrace.first})" }
+              end
+            end
+
+            payload
+          end
+
+          def send_input_chunk(chunked_payload, serialized_tags)
+            parcel = EncodedParcel.new(chunked_payload)
             request = Request.new(parcel, serialized_tags)
 
-            response = @client.send_input_payload(request)
-            unless response.ok?
-              # TODO Datadog::Core::Transport::InternalErrorResponse
-              # does not have +code+ method, what is the actual API of
-              # these response objects?
-              raise Error::AgentCommunicationError, "send_input failed: #{begin
-                response.code
-              rescue
-                "???"
-              end}: #{response.payload}"
+            client.send_request(:input, request).tap do |response|
+              if downgrade?(response)
+                downgrade!
+                return send_input_chunk(chunked_payload, serialized_tags)
+              end
             end
-          rescue Error::AgentCommunicationError
-            raise
-          # Datadog::Core::Transport does not perform any exception mapping,
-          # therefore we could have any exception here from failure to parse
-          # agent URI for example.
-          # If we ever implement retries for network errors, we should distinguish
-          # actual network errors from non-network errors that are raised by
-          # transport code.
-          rescue => exc
-            raise Error::AgentCommunicationError, "send_input failed: #{exc.class}: #{exc}"
           end
         end
       end

data/lib/datadog/di.rb

@@ -9,16 +9,13 @@ module Datadog
   #
   # @api private
   module DI
+    INSTRUMENTED_COUNTERS_LOCK = Mutex.new
+
     class << self
       def enabled?
         Datadog.configuration.dynamic_instrumentation.enabled
       end
-    end
-
-    # Expose DI to global shared objects
-    Extensions.activate!
 
-    class << self
       # This method is called from DI Remote handler to issue DI operations
       # to the probe manager (add or remove probes).
       #
@@ -31,6 +28,65 @@ module Datadog
       def component
         Datadog.send(:components).dynamic_instrumentation
       end
+
+      # Track how many outstanding instrumentations are in DI.
+      #
+      # It is hard to find the actual instrumentations - there is no
+      # method provided by Ruby to list all trace points, and we would
+      # need to manually track our instrumentation modules for method probes.
+      # Plus, tracking the modules could create active references to
+      # instrumentation, which is not desired.
+      #
+      # A simpler solution is to maintain a counter which is increased
+      # whenever a probe is installed and decreased when a probe is removed.
+      #
+      # This counter does not include pending probes - being not installed,
+      # those pose no concerns to customer applications.
+      def instrumented_count(kind = nil)
+        INSTRUMENTED_COUNTERS_LOCK.synchronize do
+          if defined?(@instrumented_count)
+            if kind
+              validate_kind!(kind)
+              @instrumented_count[kind] || 0
+            else
+              @instrumented_count.inject(0) do |sum, (_kind, count)|
+                sum + count
+              end
+            end
+          else
+            0
+          end
+        end
+      end
+
+      def instrumented_count_inc(kind)
+        validate_kind!(kind)
+        INSTRUMENTED_COUNTERS_LOCK.synchronize do
+          @instrumented_count = Hash.new(0) unless defined?(@instrumented_count)
+          @instrumented_count[kind] += 1
+        end
+      end
+
+      def instrumented_count_dec(kind)
+        validate_kind!(kind)
+        INSTRUMENTED_COUNTERS_LOCK.synchronize do
+          @instrumented_count = Hash.new(0) unless defined?(@instrumented_count)
+          if @instrumented_count[kind] <= 0
+            Datadog.logger.debug { "di: attempting to decrease instrumented count below zero for #{kind}" }
+            return
+          end
+          @instrumented_count[kind] -= 1
+        end
+      end
+
+      private def validate_kind!(kind)
+        unless %i[line method].include?(kind)
+          raise ArgumentError, "Invalid kind: #{kind}"
+        end
+      end
     end
+
+    # Expose DI to global shared objects
+    Extensions.activate!
   end
 end

data/lib/datadog/open_feature/component.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative 'transport'
+require_relative 'evaluation_engine'
+require_relative 'exposures/buffer'
+require_relative 'exposures/worker'
+require_relative 'exposures/deduplicator'
+require_relative 'exposures/reporter'
+
+module Datadog
+  module OpenFeature
+    # This class is the entry point for the OpenFeature component
+    class Component
+      attr_reader :engine
+
+      def self.build(settings, agent_settings, logger:, telemetry:)
+        return unless settings.respond_to?(:open_feature) && settings.open_feature.enabled
+
+        unless settings.respond_to?(:remote) && settings.remote.enabled
+          message = 'OpenFeature could not be enabled as Remote Configuration is currently disabled. ' \
+            'To enable Remote Configuration, see https://docs.datadoghq.com/remote_configuration/.'
+
+          logger.warn(message)
+          return
+        end
+
+        if RUBY_ENGINE != 'ruby'
+          message = 'OpenFeature could not be enabled as MRI is required, ' \
+            "but running on #{RUBY_ENGINE.inspect}"
+
+          logger.warn(message)
+          return
+        end
+
+        if (libdatadog_api_failure = Core::LIBDATADOG_API_FAILURE)
+          message = 'OpenFeature could not be enabled as `libdatadog` is not loaded: ' \
+            "#{libdatadog_api_failure.inspect}. For help solving this issue, " \
+            'please contact Datadog support at https://docs.datadoghq.com/help/.'
+
+          logger.warn(message)
+          return
+        end
+
+        new(settings, agent_settings, logger: logger, telemetry: telemetry)
+      end
+
+      def initialize(settings, agent_settings, logger:, telemetry:)
+        transport = Transport::HTTP.build(agent_settings: agent_settings, logger: logger)
+        @worker = Exposures::Worker.new(settings: settings, transport: transport, telemetry: telemetry, logger: logger)
+
+        reporter = Exposures::Reporter.new(@worker, telemetry: telemetry, logger: logger)
+        @engine = EvaluationEngine.new(reporter, telemetry: telemetry, logger: logger)
+      end
+
+      def shutdown!
+        @worker.graceful_shutdown
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/configuration.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Datadog
+  module OpenFeature
+    module Configuration
+      # A settings class for the OpenFeature component.
+      module Settings
+        def self.extended(base)
+          base = base.singleton_class unless base.is_a?(Class)
+          add_settings!(base)
+        end
+
+        def self.add_settings!(base)
+          base.class_eval do
+            settings :open_feature do
+              option :enabled do |o|
+                o.type :bool
+                o.env 'DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED'
+                o.default false
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/evaluation_engine.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+require_relative 'noop_evaluator'
+require_relative 'native_evaluator'
+require_relative 'resolution_details'
+
+module Datadog
+  module OpenFeature
+    # This class performs the evaluation of the feature flag
+    class EvaluationEngine
+      # Steep: https://github.com/soutaro/steep/issues/1880
+      ReconfigurationError = Class.new(StandardError) # steep:ignore IncompatibleAssignment
+
+      ALLOWED_TYPES = %i[boolean string number float integer object].freeze
+
+      def initialize(reporter, telemetry:, logger:)
+        @reporter = reporter
+        @telemetry = telemetry
+        @logger = logger
+
+        @evaluator = NoopEvaluator.new(nil)
+      end
+
+      def fetch_value(flag_key, default_value:, expected_type:, evaluation_context: nil)
+        unless ALLOWED_TYPES.include?(expected_type)
+          message = "unknown type #{expected_type.inspect}, allowed types #{ALLOWED_TYPES.join(", ")}"
+          return ResolutionDetails.build_error(
+            value: default_value, error_code: Ext::UNKNOWN_TYPE, error_message: message
+          )
+        end
+
+        context = evaluation_context&.fields.to_h
+        result = @evaluator.get_assignment(
+          flag_key, default_value: default_value, context: context, expected_type: expected_type
+        )
+
+        @reporter.report(result, flag_key: flag_key, context: evaluation_context)
+
+        result
+      rescue => e
+        @telemetry.report(e, description: 'OpenFeature: Failed to fetch flag value')
+
+        ResolutionDetails.build_error(
+          value: default_value, error_code: Ext::GENERAL, error_message: e.message
+        )
+      end
+
+      # NOTE: In a currect implementation configuration is expected to be a raw
+      #       JSON string containing feature flags (straight from the remote config)
+      #       in the format expected by `libdatadog` without any modifications
+      def reconfigure!(configuration)
+        if configuration.nil?
+          @logger.debug('OpenFeature: Removing configuration')
+
+          return @evaluator = NoopEvaluator.new(configuration)
+        end
+
+        @evaluator = NativeEvaluator.new(configuration)
+      rescue => e
+        message = 'OpenFeature: Failed to reconfigure, reverting to the previous configuration'
+
+        @logger.error("#{message}, #{e.class}: #{e.message}")
+        @telemetry.report(e, description: "#{message} (#{e.class})")
+
+        raise ReconfigurationError, e.message
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/exposures/batch_builder.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module OpenFeature
+    module Exposures
+      # This class builds a batch of exposures and context to be sent to the Agent
+      class BatchBuilder
+        def initialize(settings)
+          @context = build_context(settings)
+        end
+
+        def payload_for(events)
+          {
+            context: @context,
+            exposures: events
+          }
+        end
+
+        private
+
+        def build_context(settings)
+          context = {}
+          context[:env] = settings.env if settings.env
+          context[:service] = settings.service if settings.service
+          context[:version] = settings.version if settings.version
+
+          context
+        end
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/exposures/buffer.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative '../../core/buffer/cruby'
+
+module Datadog
+  module OpenFeature
+    module Exposures
+      # This class is a buffer for exposure events that evicts at random and
+      # keeps track of the number of dropped events
+      #
+      # WARNING: This class does not work as intended on JRuby
+      class Buffer < Core::Buffer::CRuby
+        DEFAULT_LIMIT = 1_000
+
+        attr_reader :dropped_count
+
+        def initialize(limit = DEFAULT_LIMIT)
+          @dropped = 0
+          @dropped_count = 0
+
+          super
+        end
+
+        protected
+
+        def drain!
+          drained = super
+
+          @dropped_count = @dropped
+          @dropped = 0
+
+          drained
+        end
+
+        def replace!(item)
+          @dropped += 1
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/exposures/deduplicator.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative '../../core/utils/lru_cache'
+
+module Datadog
+  module OpenFeature
+    module Exposures
+      # This class is a deduplication buffer based on LRU cache for exposure events
+      class Deduplicator
+        DEFAULT_CACHE_LIMIT = 1_000
+
+        def initialize(limit: DEFAULT_CACHE_LIMIT)
+          @cache = Datadog::Core::Utils::LRUCache.new(limit)
+          @mutex = Mutex.new
+        end
+
+        def duplicate?(key, value)
+          @mutex.synchronize do
+            stored = @cache[key]
+            return true if stored == value
+
+            @cache[key] = value
+          end
+
+          false
+        end
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/exposures/event.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative '../../core/utils/time'
+
+module Datadog
+  module OpenFeature
+    module Exposures
+      # A data model for an exposure event
+      module Event
+        TARGETING_KEY_FIELD = 'targeting_key'
+        ALLOWED_FIELD_TYPES = [String, Integer, Float, TrueClass, FalseClass].freeze
+
+        class << self
+          def cache_key(result, flag_key:, context:)
+            "#{flag_key}:#{context.targeting_key}"
+          end
+
+          def cache_value(result, flag_key:, context:)
+            "#{result.allocation_key}:#{result.variant}"
+          end
+
+          def build(result, flag_key:, context:)
+            {
+              timestamp: current_timestamp_ms,
+              allocation: {
+                key: result.allocation_key
+              },
+              flag: {
+                key: flag_key
+              },
+              variant: {
+                key: result.variant
+              },
+              subject: {
+                id: context.targeting_key,
+                attributes: extract_attributes(context)
+              }
+            }.freeze
+          end
+
+          private
+
+          # NOTE: We take all filds of the context that does not support nesting
+          #       and will ignore targeting key as it will be set as `subject.id`
+          def extract_attributes(context)
+            context.fields.select do |key, value|
+              next false if key == TARGETING_KEY_FIELD
+
+              ALLOWED_FIELD_TYPES.include?(value.class)
+            end
+          end
+
+          def current_timestamp_ms
+            (Datadog::Core::Utils::Time.now.to_f * 1000).to_i
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/exposures/reporter.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative 'event'
+require_relative 'deduplicator'
+
+module Datadog
+  module OpenFeature
+    module Exposures
+      # This class is responsible for reporting exposures to the Agent
+      class Reporter
+        def initialize(worker, telemetry:, logger:)
+          @worker = worker
+          @logger = logger
+          @telemetry = telemetry
+          @deduplicator = Deduplicator.new
+        end
+
+        # NOTE: Reporting expects evaluation context to be always present, but it
+        #       might be missing depending on the customer way of using flags evaluation API.
+        #       In addition to that the evaluation result must be marked for reporting.
+        def report(result, flag_key:, context:)
+          return false if context.nil?
+          return false unless result.log?
+
+          key = Event.cache_key(result, flag_key: flag_key, context: context)
+          value = Event.cache_value(result, flag_key: flag_key, context: context)
+          return false if @deduplicator.duplicate?(key, value)
+
+          event = Event.build(result, flag_key: flag_key, context: context)
+          @worker.enqueue(event)
+        rescue => e
+          @logger.debug { "OpenFeature: Failed to report resolution details: #{e.class}: #{e.message}" }
+          @telemetry.report(e, description: 'OpenFeature: Failed to report resolution details')
+
+          false
+        end
+      end
+    end
+  end
+end