cw-datadog 2.23.0.4 → 2.23.0.5

data/lib/datadog/core/environment/process.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+require_relative '../tag_normalizer'
+
+module Datadog
+  module Core
+    module Environment
+      # Retrieves process level information such that it can be attached to various payloads
+      #
+      # @api private
+      module Process
+        # This method returns a key/value part of serialized tags in the format of k1:v1,k2:v2,k3:v3
+        # @return [String] comma-separated normalized key:value pairs
+        def self.serialized
+          return @serialized if defined?(@serialized)
+          tags = []
+
+          workdir = TagNormalizer.normalize_process_value(entrypoint_workdir.to_s)
+          tags << "#{Environment::Ext::TAG_ENTRYPOINT_WORKDIR}:#{workdir}" unless workdir.empty?
+
+          entry_name = TagNormalizer.normalize_process_value(entrypoint_name.to_s)
+          tags << "#{Environment::Ext::TAG_ENTRYPOINT_NAME}:#{entry_name}" unless entry_name.empty?
+
+          basedir = TagNormalizer.normalize_process_value(entrypoint_basedir.to_s)
+          tags << "#{Environment::Ext::TAG_ENTRYPOINT_BASEDIR}:#{basedir}" unless basedir.empty?
+
+          tags << "#{Environment::Ext::TAG_ENTRYPOINT_TYPE}:#{TagNormalizer.normalize(entrypoint_type, remove_digit_start_char: false)}"
+
+          @serialized = tags.join(',').freeze
+        end
+
+        # Returns the last segment of the working directory of the process
+        # Example: /app/myapp -> myapp
+        # @return [String] the last segment of the working directory
+        def self.entrypoint_workdir
+          File.basename(Dir.pwd)
+        end
+
+        # Returns the entrypoint type of the process
+        # In Ruby, the entrypoint type is always 'script'
+        # @return [String] the type of the process, which is fixed in Ruby
+        def self.entrypoint_type
+          Environment::Ext::PROCESS_TYPE
+        end
+
+        # Returns the last segment of the base directory of the process
+        # Example 1: /bin/mybin -> mybin
+        # Example 2: ruby /test/myapp.rb -> myapp
+        # @return [String] the last segment of base directory of the script
+        #
+        # @note Determining true entrypoint name is rather complicated. This method
+        # is the initial implementation but it does not produce optimal output in all cases.
+        # For example, all Rails applications launched via `rails server` get `rails`
+        # as their entrypoint name.
+        # We might improve the behavior in the future if there is customer demand for it.
+        def self.entrypoint_name
+          File.basename($0)
+        end
+
+        # Returns the last segment of the base directory of the process
+        # Example 1: /bin/mybin -> bin
+        # Example 2: ruby /test/myapp.js -> test
+        # @return [String] the last segment of the base directory of the script
+        #
+        # @note As with entrypoint name, determining true entrypoint directory is complicated.
+        # This method has an initial implementation that does not necessarily return good
+        # results in all cases. For example, for Rails applications launched via `rails server`
+        # the entrypoint basedir is `bin` which is not very helpful.
+        # We might improve this in the future if there is customer demand.
+        def self.entrypoint_basedir
+          File.basename(File.expand_path(File.dirname($0)))
+        end
+
+        private_class_method :entrypoint_workdir, :entrypoint_type, :entrypoint_name, :entrypoint_basedir
+      end
+    end
+  end
+end

data/lib/datadog/core/feature_flags.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Datadog
+  module Core
+    # Feature flags evaluation using libdatadog
+    # The classes in this module are defined as C extensions in ext/libdatadog_api/feature_flags.c
+    #
+    # @api private
+    module FeatureFlags
+      # A top-level error raised by the extension
+      class Error < StandardError # rubocop:disable Lint/EmptyClass
+      end
+
+      # Configuration for feature flags evaluation
+      # This class is defined in the C extension
+      class Configuration # rubocop:disable Lint/EmptyClass
+      end
+
+      # Resolution details for a feature flag evaluation
+      # Base class is defined in the C extension, with Ruby methods added here
+      class ResolutionDetails
+        attr_writer :value
+
+        # Get the resolved value, with JSON parsing for object types
+        #
+        # @return [Object] The resolved value (parsed from JSON if object type)
+        # @raise [Datadog::Core::FeatureFlags::Error] If JSON parsing fails
+        def value
+          return @value if defined?(@value)
+
+          # NOTE: Raw value method call doesn't support memoization right now
+          value = raw_value
+
+          # NOTE: Lazy parsing of the JSON is a temporary solution and will be
+          #       moved into C extension
+          @value = json?(value) ? JSON.parse(value) : value
+        rescue JSON::ParserError => e
+          raise Error, "Failed to parse JSON value: #{e.class}: #{e}"
+        end
+
+        # Check if the resolution resulted in an error
+        #
+        # @return [Boolean] True if there was an error
+        def error?
+          reason == 'ERROR'
+        end
+
+        private
+
+        # NOTE: A JSON raw string will be returned by the `libdatadog` as
+        #       a Ruby String class with a flag type `:object`, otherwise it's
+        #       just a string.
+        def json?(value)
+          flag_type == :object && value.is_a?(String)
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/tag_normalizer.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require_relative 'utils'
+
+module Datadog
+  module Core
+    # @api private
+    module TagNormalizer
+      # Normalization logic used for tag keys and values that the Trace Agent has for traces
+      # Useful for ensuring that tag keys and values are normalized consistently
+      # An use case for now is Process Tags which need to be sent across various intakes (profiling, tracing, etc.) consistently
+
+      module_function
+
+      INVALID_TAG_CHARACTERS = %r{[^\p{L}0-9_\-:./]}
+      LEADING_INVALID_CHARS_NO_DIGITS = %r{\A[^\p{L}:]++}
+      LEADING_INVALID_CHARS_WITH_DIGITS = %r{\A[^\p{L}0-9:./]++}
+      MAX_BYTE_SIZE = 200 # Represents the general max tag length
+      MAX_PROCESS_VALUE_BYTE_SIZE = 100 # Represents the max tag length for process tags
+      VALID_ASCII_TAG = %r{\A[a-z:][a-z0-9:./-]*\z}
+
+      # Based on https://github.com/DataDog/datadog-agent/blob/45799c842bbd216bcda208737f9f11cade6fdd95/pkg/trace/traceutil/normalize.go#L131
+      # Specifically for general normalization:
+      # - Must be valid UTF-8
+      # - Invalid characters are replaced with an underscore
+      # - Leading non-letter characters are removed but colons are kept
+      # - Trailing non-letter characters are removed
+      # - Trailing underscores are removed
+      # - Consecutive underscores are merged into a single underscore
+      # - Maximum length is 200 characters
+      # If it's a tag value, allow it to start with a digit
+      # @param original_value [String] The original string
+      # @param remove_digit_start_char [Boolean] - whether to remove the leading digit (currently only used for tag values)
+      # @return [String] The normalized string
+      def self.normalize(original_value, remove_digit_start_char: false)
+        # DEV-3.0: Ideally this encode call should be replaced with Datadog::Core::Utils.utf8_encode once it
+        #          is safe to modify the default behavior.
+        value = original_value.to_s.encode('UTF-8', invalid: :replace, undef: :replace)
+        value.strip!
+        return "" if value.empty?
+
+        return value if value.bytesize <= MAX_BYTE_SIZE &&
+          value.match?(VALID_ASCII_TAG)
+
+        if value.bytesize > MAX_BYTE_SIZE
+          value = value.byteslice(0, MAX_BYTE_SIZE)
+          value.scrub!("")
+        end
+
+        value.downcase!
+        value.gsub!(INVALID_TAG_CHARACTERS, '_')
+
+        # The Trace Agent allows tag values to start with a number so this logic is here too
+        leading_invalid_regex = remove_digit_start_char ? LEADING_INVALID_CHARS_NO_DIGITS : LEADING_INVALID_CHARS_WITH_DIGITS
+        value.sub!(leading_invalid_regex, "")
+
+        value.squeeze!('_') if value.include?('__')
+        value.delete_suffix!('_')
+
+        value
+      end
+
+      # Process tags values follow an additional piece of normalization:
+      # - must not be more than 100 bytes
+      # - and must not contain colons
+      # @param value [String] The original string
+      # @return [String] The normalized string
+      def self.normalize_process_value(value)
+        value = normalize(value)
+        return value if value.empty?
+
+        value.tr!(':', '_')
+        value.squeeze!('_') if value.include?('__')
+
+        if value.bytesize > MAX_PROCESS_VALUE_BYTE_SIZE
+          value = value.byteslice(0, MAX_PROCESS_VALUE_BYTE_SIZE) || value
+          value.scrub!("")
+        end
+
+        value
+      end
+    end
+  end
+end

data/lib/datadog/core/transport/http/adapters/net.rb

@@ -81,6 +81,14 @@ module Datadog
                 http.request(post)
               end
 
+              # Debug log: response result
+              Datadog.logger.debug do
+                response_body = http_response.body.to_s
+                # 截断过长的响应体
+                truncated_body = response_body.length > 500 ? "#{response_body[0..500]}..." : response_body
+                "[Trace Transport] Flushing trace response: POST.uri =#{net_http_path_from_env(env)}, body=#{truncated_body}"
+              end if defined?(Datadog.logger)
+
               # Build and return response
               Response.new(http_response)
             end

data/lib/datadog/core/utils/array.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Utils
+      # Common array-related utility functions.
+      module Array
+        def self.filter_map(array, &block)
+          if array.respond_to?(:filter_map)
+            # DEV Supported since Ruby 2.7, saves an intermediate object creation
+            array.filter_map(&block)
+          elsif array.is_a?(Enumerator::Lazy)
+            # You would think that .compact would work here, but it does not:
+            # the result of .map could be an Enumerator::Lazy instance which
+            # does not implement #compact on Ruby 2.5/2.6.
+            array.map(&block).reject do |item|
+              item.nil?
+            end
+          else
+            array.each_with_object([]) do |item, memo|
+              new_item = block.call(item)
+              memo.push(new_item) unless new_item.nil?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/utils.rb

@@ -38,6 +38,8 @@ module Datadog
 
       # Ensure `str` is a valid UTF-8, ready to be
       # sent through the tracer transport.
+      # DEV-3.0: This method should unconditionally handle invalid byte sequences
+      # DEV-3.0: and return a safe string to display.
       #
       # @param [String,#to_s] str object to be converted to a UTF-8 string
       # @param [Boolean] binary whether to expect binary data in the `str` parameter

data/lib/datadog/data_streams/processor.rb

@@ -475,7 +475,7 @@ module Datadog
         @consumer_stats.map do |stat|
           {
             'Tags' => [
-              'type:kafka_consume',
+              'type:kafka_commit',
               "topic:#{stat[:topic]}",
               "partition:#{stat[:partition]}"
             ],

data/lib/datadog/di/transport/http.rb

@@ -40,9 +40,13 @@ module Datadog
           api_version: nil,
           headers: nil
         )
-          Core::Transport::HTTP.build(api_instance_class: Input::API::Instance,
+          Core::Transport::HTTP.build(
+            api_instance_class: Input::API::Instance,
             logger: logger,
-            agent_settings: agent_settings, api_version: api_version, headers: headers) do |transport|
+            agent_settings: agent_settings,
+            api_version: api_version,
+            headers: headers,
+          ) do |transport|
             apis = API.defaults
 
             transport.api API::INPUT, apis[API::INPUT]

data/lib/datadog/di/transport/input.rb

@@ -1,6 +1,11 @@
 # frozen_string_literal: true
 
+require_relative '../../core/chunker'
+require_relative '../../core/encoding'
+require_relative '../../core/tag_builder'
 require_relative '../../core/transport/parcel'
+require_relative '../../core/transport/request'
+require_relative '../error'
 require_relative 'http/input'
 
 module Datadog
@@ -24,6 +29,25 @@ module Datadog
         class Transport
           attr_reader :client, :apis, :default_api, :current_api_id, :logger
 
+          # The limit on an individual snapshot payload, aka "log line",
+          # is 1 MB.
+          #
+          # TODO There is an RFC for snapshot pruning that should be
+          # implemented to reduce the size of snapshots to be below this
+          # limit, so that we can send a portion of the captured data
+          # rather than dropping the snapshot entirely.
+          MAX_SERIALIZED_SNAPSHOT_SIZE = 1024 * 1024
+
+          # The maximum chunk (batch) size that intake permits is 5 MB.
+          #
+          # Two bytes are for the [ and ] of JSON array syntax.
+          MAX_CHUNK_SIZE = 5 * 1024 * 1024 - 2
+
+          # Try to send smaller payloads to avoid large network requests.
+          # If a payload is larger than default chunk size but is under the
+          # max chunk size, it will still get sent out.
+          DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024
+
           def initialize(apis, default_api, logger:)
             @apis = apis
             @logger = logger
@@ -36,9 +60,45 @@ module Datadog
           end
 
           def send_input(payload, tags)
-            json = JSON.dump(payload)
-            parcel = EncodedParcel.new(json)
+            # Tags are the same for all chunks, serialize them one time.
             serialized_tags = Core::TagBuilder.serialize_tags(tags)
+
+            encoder = Core::Encoding::JSONEncoder
+            encoded_snapshots = Core::Utils::Array.filter_map(payload) do |snapshot|
+              encoded = encoder.encode(snapshot)
+              if encoded.length > MAX_SERIALIZED_SNAPSHOT_SIZE
+                # Drop the snapshot.
+                # TODO report via telemetry metric?
+                logger.debug { "di: dropping too big snapshot" }
+                nil
+              else
+                encoded
+              end
+            end
+
+            Datadog::Core::Chunker.chunk_by_size(
+              encoded_snapshots, DEFAULT_CHUNK_SIZE,
+            ).each do |chunk|
+              # We drop snapshots that are too big earlier.
+              # The limit on chunked payload length here is greater
+              # than the limit on snapshot size, therefore no chunks
+              # can exceed limits here.
+              chunked_payload = encoder.join(chunk)
+
+              # We need to rescue exceptions for each chunk so that
+              # subsequent chunks are attempted to be sent.
+              begin
+                send_input_chunk(chunked_payload, serialized_tags)
+              rescue => exc
+                logger.debug { "di: failed to send snapshot chunk: #{exc.class}: #{exc} (at #{exc.backtrace.first})" }
+              end
+            end
+
+            payload
+          end
+
+          def send_input_chunk(chunked_payload, serialized_tags)
+            parcel = EncodedParcel.new(chunked_payload)
             request = Request.new(parcel, serialized_tags)
 
             response = @client.send_input_payload(request)

data/lib/datadog/open_feature/evaluation_engine.rb

@@ -2,6 +2,7 @@
 
 require_relative 'ext'
 require_relative 'noop_evaluator'
+require_relative 'native_evaluator'
 require_relative 'resolution_details'
 
 module Datadog
@@ -10,7 +11,7 @@ module Datadog
     class EvaluationEngine
       ReconfigurationError = Class.new(StandardError)
 
-      ALLOWED_TYPES = %w[boolean string number float integer object].freeze
+      ALLOWED_TYPES = %i[boolean string number float integer object].freeze
 
       def initialize(reporter, telemetry:, logger:)
         @reporter = reporter
@@ -20,7 +21,7 @@ module Datadog
         @evaluator = NoopEvaluator.new(nil)
       end
 
-      def fetch_value(flag_key:, default_value:, expected_type:, evaluation_context: nil)
+      def fetch_value(flag_key, default_value:, expected_type:, evaluation_context: nil)
         unless ALLOWED_TYPES.include?(expected_type)
           message = "unknown type #{expected_type.inspect}, allowed types #{ALLOWED_TYPES.join(", ")}"
           return ResolutionDetails.build_error(
@@ -28,8 +29,10 @@ module Datadog
           )
         end
 
-        context = evaluation_context&.fields || {}
-        result = @evaluator.get_assignment(flag_key, default_value, context, expected_type)
+        context = evaluation_context&.fields.to_h
+        result = @evaluator.get_assignment(
+          flag_key, default_value: default_value, context: context, expected_type: expected_type
+        )
 
         @reporter.report(result, flag_key: flag_key, context: evaluation_context)
 
@@ -38,19 +41,26 @@ module Datadog
         @telemetry.report(e, description: 'OpenFeature: Failed to fetch flag value')
 
         ResolutionDetails.build_error(
-          value: default_value, error_code: Ext::PROVIDER_FATAL, error_message: e.message
+          value: default_value, error_code: Ext::GENERAL, error_message: e.message
         )
       end
 
+      # NOTE: In a currect implementation configuration is expected to be a raw
+      #       JSON string containing feature flags (straight from the remote config)
+      #       in the format expected by `libdatadog` without any modifications
       def reconfigure!(configuration)
-        @logger.debug('OpenFeature: Removing configuration') if configuration.nil?
+        if configuration.nil?
+          @logger.debug('OpenFeature: Removing configuration')
+
+          return @evaluator = NoopEvaluator.new(configuration)
+        end
 
-        @evaluator = NoopEvaluator.new(configuration)
+        @evaluator = NativeEvaluator.new(configuration)
       rescue => e
         message = 'OpenFeature: Failed to reconfigure, reverting to the previous configuration'
 
-        @logger.error("#{message}, error #{e.inspect}")
-        @telemetry.report(e, description: message)
+        @logger.error("#{message}, #{e.class}: #{e.message}")
+        @telemetry.report(e, description: "#{message} (#{e.class})")
 
         raise ReconfigurationError, e.message
       end

data/lib/datadog/open_feature/ext.rb

@@ -6,6 +6,7 @@ module Datadog
       ERROR = 'ERROR'
       INITIALIZING = 'INITIALIZING'
       UNKNOWN_TYPE = 'UNKNOWN_TYPE'
+      GENERAL = 'GENERAL'
       PROVIDER_FATAL = 'PROVIDER_FATAL'
       PROVIDER_NOT_READY = 'PROVIDER_NOT_READY'
     end

data/lib/datadog/open_feature/native_evaluator.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative '../core/feature_flags'
+
+module Datadog
+  module OpenFeature
+    # This class is an interface of evaluation logic using native extension
+    class NativeEvaluator
+      # NOTE: In a currect implementation configuration is expected to be a raw
+      #       JSON string containing feature flags (straight from the remote config)
+      #       in the format expected by `libdatadog` without any modifications
+      def initialize(configuration)
+        @configuration = Core::FeatureFlags::Configuration.new(configuration)
+      end
+
+      # Returns the assignment for a given flag key based on the feature flags
+      # configuration
+      #
+      # @param flag_key [String] The key of the feature flag
+      # @param default_value [Object] The default value to return if the flag is
+      #                              not found or evaluation itself fails
+      # @param expected_type [Symbol] The expected type of the flag
+      # @param context [Hash] The context of the evaluation, containing targeting key
+      #                       and other attributes
+      #
+      # @return [Core::FeatureFlags::ResolutionDetails] The assignment for the flag
+      def get_assignment(flag_key, default_value:, expected_type:, context:)
+        result = @configuration.get_assignment(flag_key, expected_type, context)
+
+        # NOTE: This is a special case when we need to fallback to the default
+        #       value, even tho the evaluation itself doesn't produce an error
+        #       resolution details
+        result.value = default_value if result.variant.nil?
+        result
+      end
+    end
+  end
+end

data/lib/datadog/open_feature/noop_evaluator.rb

@@ -11,14 +11,14 @@ module Datadog
         # no-op
       end
 
-      def get_assignment(_flag_key, default_value, _context, _expected_type)
+      def get_assignment(_flag_key, default_value:, context:, expected_type:)
         ResolutionDetails.new(
           value: default_value,
           log?: false,
           error?: true,
           error_code: Ext::PROVIDER_NOT_READY,
-          error_message: 'Waiting for universal flag configuration',
-          reason: Ext::INITIALIZING
+          error_message: 'Waiting for flags configuration',
+          reason: Ext::ERROR
         )
       end
     end

data/lib/datadog/open_feature/provider.rb

@@ -68,27 +68,27 @@ module Datadog
       end
 
       def fetch_boolean_value(flag_key:, default_value:, evaluation_context: nil)
-        evaluate(flag_key, default_value: default_value, expected_type: 'boolean', evaluation_context: evaluation_context)
+        evaluate(flag_key, default_value: default_value, expected_type: :boolean, evaluation_context: evaluation_context)
       end
 
       def fetch_string_value(flag_key:, default_value:, evaluation_context: nil)
-        evaluate(flag_key, default_value: default_value, expected_type: 'string', evaluation_context: evaluation_context)
+        evaluate(flag_key, default_value: default_value, expected_type: :string, evaluation_context: evaluation_context)
       end
 
       def fetch_number_value(flag_key:, default_value:, evaluation_context: nil)
-        evaluate(flag_key, default_value: default_value, expected_type: 'number', evaluation_context: evaluation_context)
+        evaluate(flag_key, default_value: default_value, expected_type: :number, evaluation_context: evaluation_context)
       end
 
       def fetch_integer_value(flag_key:, default_value:, evaluation_context: nil)
-        evaluate(flag_key, default_value: default_value, expected_type: 'integer', evaluation_context: evaluation_context)
+        evaluate(flag_key, default_value: default_value, expected_type: :integer, evaluation_context: evaluation_context)
       end
 
       def fetch_float_value(flag_key:, default_value:, evaluation_context: nil)
-        evaluate(flag_key, default_value: default_value, expected_type: 'float', evaluation_context: evaluation_context)
+        evaluate(flag_key, default_value: default_value, expected_type: :float, evaluation_context: evaluation_context)
       end
 
       def fetch_object_value(flag_key:, default_value:, evaluation_context: nil)
-        evaluate(flag_key, default_value: default_value, expected_type: 'object', evaluation_context: evaluation_context)
+        evaluate(flag_key, default_value: default_value, expected_type: :object, evaluation_context: evaluation_context)
       end
 
       private
@@ -98,7 +98,7 @@ module Datadog
         return component_not_configured_default(default_value) if engine.nil?
 
         result = engine.fetch_value(
-          flag_key: flag_key,
+          flag_key,
           default_value: default_value,
           expected_type: expected_type,
           evaluation_context: evaluation_context
@@ -106,7 +106,7 @@ module Datadog
 
         if result.error?
           return ::OpenFeature::SDK::Provider::ResolutionDetails.new(
-            value: result.value,
+            value: default_value,
             error_code: result.error_code,
             error_message: result.error_message,
             reason: result.reason
@@ -119,6 +119,13 @@ module Datadog
           reason: result.reason,
           flag_metadata: result.flag_metadata
         )
+      rescue => e
+        ::OpenFeature::SDK::Provider::ResolutionDetails.new(
+          value: default_value,
+          error_code: Ext::GENERAL,
+          error_message: "#{e.class}: #{e.message}",
+          reason: Ext::ERROR
+        )
       end
 
       def component_not_configured_default(value)

data/lib/datadog/open_feature/remote.rb

@@ -25,7 +25,7 @@ module Datadog
           matcher = Core::Remote::Dispatcher::Matcher::Product.new(FFE_PRODUCTS)
           receiver = Core::Remote::Dispatcher::Receiver.new(matcher) do |repository, changes|
             engine = OpenFeature.engine
-            break unless engine
+            next unless engine
 
             changes.each do |change|
               content = repository[change.path]