datadog 2.22.0 → 2.23.0

data/lib/datadog/di/probe_notification_builder.rb

@@ -87,10 +87,41 @@ module Datadog
           end
         end
 
+        message = nil
+        evaluation_errors = []
+        if segments = probe.template_segments
+          message, evaluation_errors = evaluate_template(segments, context)
+        end
+        build_snapshot_base(context,
+          evaluation_errors: evaluation_errors, message: message,
+          captures: captures)
+      end
+
+      def build_condition_evaluation_failed(context, expression, exception)
+        error = {
+          message: "#{exception.class}: #{exception}",
+          expr: expression.dsl_expr,
+        }
+        build_snapshot_base(context, evaluation_errors: [error])
+      end
+
+      private
+
+      def build_snapshot_base(context, evaluation_errors: [], captures: nil, message: nil)
+        probe = context.probe
+
+        timestamp = timestamp_now
+        duration = context.duration
+
         location = if probe.line?
           {
             file: context.path,
-            lines: [probe.line_no],
+            # Line numbers are required to be strings by the
+            # system tests schema.
+            # Backend I think accepts them also as integers, but some
+            # other languages send strings and we decided to require
+            # strings for everyone.
+            lines: [probe.line_no.to_s],
           }
         elsif probe.method?
           {
@@ -103,28 +134,38 @@ module Datadog
           format_caller_locations(caller_locations)
         end
 
-        timestamp = timestamp_now
-        message = nil
-        evaluation_errors = []
-        if segments = probe.template_segments
-          message, evaluation_errors = evaluate_template(segments, context)
-        end
-        duration = context.duration
         {
           service: settings.service,
-          "debugger.snapshot": {
-            id: SecureRandom.uuid,
-            timestamp: timestamp,
-            evaluationErrors: evaluation_errors,
-            probe: {
-              id: probe.id,
-              version: 0,
-              location: location,
+          debugger: {
+            type: 'snapshot',
+            # Product can have three values: di, ld, er.
+            # We do not currently implement exception replay.
+            # There is currently no specification, and no consensus, for
+            # when product should be di (dynamic instrumentation) and when
+            # it should be ld (live debugger). I thought the backend was
+            # supposed to provide this in probe specification via remote
+            # config, but apparently this is not the case and the expectation
+            # is that the library figures out the product via heuristics,
+            # except there is currently no consensus on said heuristics.
+            # .NET always sends ld, other languages send nothing at the moment.
+            # Don't send anything for the time being.
+            #product: 'di/ld',
+            snapshot: {
+              id: SecureRandom.uuid,
+              timestamp: timestamp,
+              evaluationErrors: evaluation_errors,
+              probe: {
+                id: probe.id,
+                version: 0,
+                location: location,
+              },
+              language: 'ruby',
+              # TODO add test coverage for callers being nil
+              stack: stack,
+              # System tests schema validation requires captures to
+              # always be present
+              captures: captures || {},
             },
-            language: 'ruby',
-            # TODO add test coverage for callers being nil
-            stack: stack,
-            captures: captures,
           },
           # In python tracer duration is under debugger.snapshot,
           # but UI appears to expect it here at top level.
@@ -132,7 +173,7 @@ module Datadog
           host: nil,
           logger: {
             name: probe.file,
-            method: probe.method_name || 'no_method',
+            method: probe.method_name,
             thread_name: Thread.current.name,
             # Dynamic instrumentation currently does not need thread_id for
             # anything. It can be sent if a customer requests it at which point
@@ -150,8 +191,6 @@ module Datadog
         }
       end
 
-      private
-
       def build_status(probe, message:, status:)
         {
           service: settings.service,

data/lib/datadog/di/proc_responder.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module DI
+    # An adapter to convert procs to responders.
+    #
+    # Used in test suite and benchmarks.
+    #
+    # @api private
+    class ProcResponder
+      def initialize(executed_proc, failed_proc = nil)
+        @executed_proc = executed_proc
+        @failed_proc = failed_proc
+      end
+
+      attr_reader :executed_proc
+      attr_reader :failed_proc
+
+      def probe_executed_callback(context)
+        executed_proc.call(context)
+      end
+
+      def probe_condition_evaluation_failed_callback(context, exc)
+        if failed_proc.nil?
+          raise NotImplementedError, "Failed proc not provided"
+        end
+
+        failed_proc.call(context, exc)
+      end
+    end
+  end
+end

data/lib/datadog/di/transport/diagnostics.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../core/transport/parcel'
-require_relative 'http/client'
+require_relative 'http/diagnostics'
 
 module Datadog
   module DI
@@ -21,7 +21,7 @@ module Datadog
             @apis = apis
             @logger = logger
 
-            @client = HTTP::Client.new(current_api, logger: logger)
+            @client = DI::Transport::HTTP::Diagnostics::Client.new(current_api, logger: logger)
           end
 
           def current_api

data/lib/datadog/di/transport/http/diagnostics.rb

@@ -2,14 +2,14 @@
 
 require_relative '../../../core/transport/http/api/instance'
 require_relative '../../../core/transport/http/api/spec'
-require_relative 'client'
+require_relative '../../../core/transport/http/client'
 
 module Datadog
   module DI
     module Transport
       module HTTP
         module Diagnostics
-          module Client
+          class Client < Core::Transport::HTTP::Client
             def send_diagnostics_payload(request)
               send_request(request) do |api, env|
                 api.send_diagnostics(env)
@@ -57,8 +57,6 @@ module Datadog
             end
           end
         end
-
-        HTTP::Client.include(Diagnostics::Client)
       end
     end
   end

data/lib/datadog/di/transport/http/input.rb

@@ -2,14 +2,14 @@
 
 require_relative '../../../core/transport/http/api/instance'
 require_relative '../../../core/transport/http/api/spec'
-require_relative 'client'
+require_relative '../../../core/transport/http/client'
 
 module Datadog
   module DI
     module Transport
       module HTTP
         module Input
-          module Client
+          class Client < Core::Transport::HTTP::Client
             def send_input_payload(request)
               send_request(request) do |api, env|
                 api.send_input(env)
@@ -69,8 +69,6 @@ module Datadog
             end
           end
         end
-
-        HTTP::Client.include(Input::Client)
       end
     end
   end

data/lib/datadog/di/transport/http.rb

@@ -40,9 +40,13 @@ module Datadog
           api_version: nil,
           headers: nil
         )
-          Core::Transport::HTTP.build(api_instance_class: Input::API::Instance,
+          Core::Transport::HTTP.build(
+            api_instance_class: Input::API::Instance,
             logger: logger,
-            agent_settings: agent_settings, api_version: api_version, headers: headers) do |transport|
+            agent_settings: agent_settings,
+            api_version: api_version,
+            headers: headers,
+          ) do |transport|
             apis = API.defaults
 
             transport.api API::INPUT, apis[API::INPUT]

data/lib/datadog/di/transport/input.rb

@@ -1,7 +1,12 @@
 # frozen_string_literal: true
 
+require_relative '../../core/chunker'
+require_relative '../../core/encoding'
+require_relative '../../core/tag_builder'
 require_relative '../../core/transport/parcel'
-require_relative 'http/client'
+require_relative '../../core/transport/request'
+require_relative '../error'
+require_relative 'http/input'
 
 module Datadog
   module DI
@@ -24,11 +29,30 @@ module Datadog
         class Transport
           attr_reader :client, :apis, :default_api, :current_api_id, :logger
 
+          # The limit on an individual snapshot payload, aka "log line",
+          # is 1 MB.
+          #
+          # TODO There is an RFC for snapshot pruning that should be
+          # implemented to reduce the size of snapshots to be below this
+          # limit, so that we can send a portion of the captured data
+          # rather than dropping the snapshot entirely.
+          MAX_SERIALIZED_SNAPSHOT_SIZE = 1024 * 1024
+
+          # The maximum chunk (batch) size that intake permits is 5 MB.
+          #
+          # Two bytes are for the [ and ] of JSON array syntax.
+          MAX_CHUNK_SIZE = 5 * 1024 * 1024 - 2
+
+          # Try to send smaller payloads to avoid large network requests.
+          # If a payload is larger than default chunk size but is under the
+          # max chunk size, it will still get sent out.
+          DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024
+
           def initialize(apis, default_api, logger:)
             @apis = apis
             @logger = logger
 
-            @client = HTTP::Client.new(current_api, logger: logger)
+            @client = DI::Transport::HTTP::Input::Client.new(current_api, logger: logger)
           end
 
           def current_api
@@ -36,9 +60,45 @@ module Datadog
           end
 
           def send_input(payload, tags)
-            json = JSON.dump(payload)
-            parcel = EncodedParcel.new(json)
+            # Tags are the same for all chunks, serialize them one time.
             serialized_tags = Core::TagBuilder.serialize_tags(tags)
+
+            encoder = Core::Encoding::JSONEncoder
+            encoded_snapshots = Core::Utils::Array.filter_map(payload) do |snapshot|
+              encoded = encoder.encode(snapshot)
+              if encoded.length > MAX_SERIALIZED_SNAPSHOT_SIZE
+                # Drop the snapshot.
+                # TODO report via telemetry metric?
+                logger.debug { "di: dropping too big snapshot" }
+                nil
+              else
+                encoded
+              end
+            end
+
+            Datadog::Core::Chunker.chunk_by_size(
+              encoded_snapshots, DEFAULT_CHUNK_SIZE,
+            ).each do |chunk|
+              # We drop snapshots that are too big earlier.
+              # The limit on chunked payload length here is greater
+              # than the limit on snapshot size, therefore no chunks
+              # can exceed limits here.
+              chunked_payload = encoder.join(chunk)
+
+              # We need to rescue exceptions for each chunk so that
+              # subsequent chunks are attempted to be sent.
+              begin
+                send_input_chunk(chunked_payload, serialized_tags)
+              rescue => exc
+                logger.debug { "di: failed to send snapshot chunk: #{exc.class}: #{exc} (at #{exc.backtrace.first})" }
+              end
+            end
+
+            payload
+          end
+
+          def send_input_chunk(chunked_payload, serialized_tags)
+            parcel = EncodedParcel.new(chunked_payload)
             request = Request.new(parcel, serialized_tags)
 
             response = @client.send_input_payload(request)

data/lib/datadog/open_feature/component.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative 'transport'
+require_relative 'evaluation_engine'
+require_relative 'exposures/buffer'
+require_relative 'exposures/worker'
+require_relative 'exposures/deduplicator'
+require_relative 'exposures/reporter'
+
+module Datadog
+  module OpenFeature
+    # This class is the entry point for the OpenFeature component
+    class Component
+      attr_reader :engine
+
+      def self.build(settings, agent_settings, logger:, telemetry:)
+        return unless settings.respond_to?(:open_feature) && settings.open_feature.enabled
+
+        unless settings.respond_to?(:remote) && settings.remote.enabled
+          message = 'OpenFeature could not be enabled as Remote Configuration is currently disabled. ' \
+            'To enable Remote Configuration, see https://docs.datadoghq.com/remote_configuration/.'
+
+          logger.warn(message)
+          return
+        end
+
+        if RUBY_ENGINE != 'ruby'
+          message = 'OpenFeature could not be enabled as MRI is required, ' \
+            "but running on #{RUBY_ENGINE.inspect}"
+
+          logger.warn(message)
+          return
+        end
+
+        if (libdatadog_api_failure = Core::LIBDATADOG_API_FAILURE)
+          message = 'OpenFeature could not be enabled as `libdatadog` is not loaded: ' \
+            "#{libdatadog_api_failure.inspect}. For help solving this issue, " \
+            'please contact Datadog support at https://docs.datadoghq.com/help/.'
+
+          logger.warn(message)
+          return
+        end
+
+        new(settings, agent_settings, logger: logger, telemetry: telemetry)
+      end
+
+      def initialize(settings, agent_settings, logger:, telemetry:)
+        transport = Transport::HTTP.build(agent_settings: agent_settings, logger: logger)
+        @worker = Exposures::Worker.new(settings: settings, transport: transport, telemetry: telemetry, logger: logger)
+
+        reporter = Exposures::Reporter.new(@worker, telemetry: telemetry, logger: logger)
+        @engine = EvaluationEngine.new(reporter, telemetry: telemetry, logger: logger)
+      end
+
+      def shutdown!
+        @worker.graceful_shutdown
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/configuration.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Datadog
+  module OpenFeature
+    module Configuration
+      # A settings class for the OpenFeature component.
+      module Settings
+        def self.extended(base)
+          base = base.singleton_class unless base.is_a?(Class)
+          add_settings!(base)
+        end
+
+        def self.add_settings!(base)
+          base.class_eval do
+            settings :open_feature do
+              option :enabled do |o|
+                o.type :bool
+                o.env 'DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED'
+                o.default false
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/evaluation_engine.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+require_relative 'noop_evaluator'
+require_relative 'native_evaluator'
+require_relative 'resolution_details'
+
+module Datadog
+  module OpenFeature
+    # This class performs the evaluation of the feature flag
+    class EvaluationEngine
+      ReconfigurationError = Class.new(StandardError)
+
+      ALLOWED_TYPES = %i[boolean string number float integer object].freeze
+
+      def initialize(reporter, telemetry:, logger:)
+        @reporter = reporter
+        @telemetry = telemetry
+        @logger = logger
+
+        @evaluator = NoopEvaluator.new(nil)
+      end
+
+      def fetch_value(flag_key, default_value:, expected_type:, evaluation_context: nil)
+        unless ALLOWED_TYPES.include?(expected_type)
+          message = "unknown type #{expected_type.inspect}, allowed types #{ALLOWED_TYPES.join(", ")}"
+          return ResolutionDetails.build_error(
+            value: default_value, error_code: Ext::UNKNOWN_TYPE, error_message: message
+          )
+        end
+
+        context = evaluation_context&.fields.to_h
+        result = @evaluator.get_assignment(
+          flag_key, default_value: default_value, context: context, expected_type: expected_type
+        )
+
+        @reporter.report(result, flag_key: flag_key, context: evaluation_context)
+
+        result
+      rescue => e
+        @telemetry.report(e, description: 'OpenFeature: Failed to fetch flag value')
+
+        ResolutionDetails.build_error(
+          value: default_value, error_code: Ext::GENERAL, error_message: e.message
+        )
+      end
+
+      # NOTE: In a currect implementation configuration is expected to be a raw
+      #       JSON string containing feature flags (straight from the remote config)
+      #       in the format expected by `libdatadog` without any modifications
+      def reconfigure!(configuration)
+        if configuration.nil?
+          @logger.debug('OpenFeature: Removing configuration')
+
+          return @evaluator = NoopEvaluator.new(configuration)
+        end
+
+        @evaluator = NativeEvaluator.new(configuration)
+      rescue => e
+        message = 'OpenFeature: Failed to reconfigure, reverting to the previous configuration'
+
+        @logger.error("#{message}, #{e.class}: #{e.message}")
+        @telemetry.report(e, description: "#{message} (#{e.class})")
+
+        raise ReconfigurationError, e.message
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/exposures/batch_builder.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module OpenFeature
+    module Exposures
+      # This class builds a batch of exposures and context to be sent to the Agent
+      class BatchBuilder
+        def initialize(settings)
+          @context = build_context(settings)
+        end
+
+        def payload_for(events)
+          {
+            context: @context,
+            exposures: events
+          }
+        end
+
+        private
+
+        def build_context(settings)
+          context = {}
+          context[:env] = settings.env if settings.env
+          context[:service] = settings.service if settings.service
+          context[:version] = settings.version if settings.version
+
+          context
+        end
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/exposures/buffer.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative '../../core/buffer/cruby'
+
+module Datadog
+  module OpenFeature
+    module Exposures
+      # This class is a buffer for exposure events that evicts at random and
+      # keeps track of the number of dropped events
+      #
+      # WARNING: This class does not work as intended on JRuby
+      class Buffer < Core::Buffer::CRuby
+        DEFAULT_LIMIT = 1_000
+
+        attr_reader :dropped_count
+
+        def initialize(limit = DEFAULT_LIMIT)
+          @dropped = 0
+          @dropped_count = 0
+
+          super
+        end
+
+        protected
+
+        def drain!
+          drained = super
+
+          @dropped_count = @dropped
+          @dropped = 0
+
+          drained
+        end
+
+        def replace!(item)
+          @dropped += 1
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/exposures/deduplicator.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative '../../core/utils/lru_cache'
+
+module Datadog
+  module OpenFeature
+    module Exposures
+      # This class is a deduplication buffer based on LRU cache for exposure events
+      class Deduplicator
+        DEFAULT_CACHE_LIMIT = 1_000
+
+        def initialize(limit: DEFAULT_CACHE_LIMIT)
+          @cache = Datadog::Core::Utils::LRUCache.new(limit)
+          @mutex = Mutex.new
+        end
+
+        def duplicate?(key, value)
+          @mutex.synchronize do
+            stored = @cache[key]
+            return true if stored == value
+
+            @cache[key] = value
+          end
+
+          false
+        end
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/exposures/event.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative '../../core/utils/time'
+
+module Datadog
+  module OpenFeature
+    module Exposures
+      # A data model for an exposure event
+      module Event
+        TARGETING_KEY_FIELD = 'targeting_key'
+        ALLOWED_FIELD_TYPES = [String, Integer, Float, TrueClass, FalseClass].freeze
+
+        class << self
+          def cache_key(result, flag_key:, context:)
+            "#{flag_key}:#{context.targeting_key}"
+          end
+
+          def cache_value(result, flag_key:, context:)
+            "#{result.allocation_key}:#{result.variant}"
+          end
+
+          def build(result, flag_key:, context:)
+            {
+              timestamp: current_timestamp_ms,
+              allocation: {
+                key: result.allocation_key
+              },
+              flag: {
+                key: flag_key
+              },
+              variant: {
+                key: result.variant
+              },
+              subject: {
+                id: context.targeting_key,
+                attributes: extract_attributes(context)
+              }
+            }.freeze
+          end
+
+          private
+
+          # NOTE: We take all filds of the context that does not support nesting
+          #       and will ignore targeting key as it will be set as `subject.id`
+          def extract_attributes(context)
+            context.fields.select do |key, value|
+              next false if key == TARGETING_KEY_FIELD
+
+              ALLOWED_FIELD_TYPES.include?(value.class)
+            end
+          end
+
+          def current_timestamp_ms
+            (Datadog::Core::Utils::Time.now.to_f * 1000).to_i
+          end
+        end
+      end
+    end
+  end
+end