datadog 2.13.0 → 2.15.0

data/lib/datadog/appsec/component.rb

@@ -12,7 +12,25 @@ module Datadog
       class << self
         def build_appsec_component(settings, telemetry:)
           return if !settings.respond_to?(:appsec) || !settings.appsec.enabled
-          return if incompatible_ffi_version?
+
+          ffi_version = Gem.loaded_specs['ffi'] && Gem.loaded_specs['ffi'].version
+          unless ffi_version
+            Datadog.logger.warn('FFI gem is not loaded, AppSec will be disabled.')
+            telemetry.error('AppSec: Component not loaded, due to missing FFI gem')
+
+            return
+          end
+
+          if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.3') && ffi_version < Gem::Version.new('1.16.0')
+            Datadog.logger.warn(
+              'AppSec is not supported in Ruby versions above 3.3.0 when using `ffi` versions older than 1.16.0, ' \
+              'and will be forcibly disabled due to a memory leak in `ffi`. ' \
+              'Please upgrade your `ffi` version to 1.16.0 or higher.'
+            )
+            telemetry.error('AppSec: Component not loaded, ffi version is leaky with ruby > 3.3.0')
+
+            return
+          end
 
           processor = create_processor(settings, telemetry)
 
@@ -29,22 +47,6 @@ module Datadog
 
         private
 
-        def incompatible_ffi_version?
-          ffi_version = Gem.loaded_specs['ffi'] && Gem.loaded_specs['ffi'].version
-          return true unless ffi_version
-
-          return false unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.3') &&
-            ffi_version < Gem::Version.new('1.16.0')
-
-          Datadog.logger.warn(
-            'AppSec is not supported in Ruby versions above 3.3.0 when using `ffi` versions older than 1.16.0, ' \
-            'and will be forcibly disabled due to a memory leak in `ffi`. ' \
-            'Please upgrade your `ffi` version to 1.16.0 or higher.'
-          )
-
-          true
-        end
-
         def create_processor(settings, telemetry)
           rules = AppSec::Processor::RuleLoader.load_rules(
             telemetry: telemetry,

data/lib/datadog/appsec/compressed_json.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'zlib'
+require 'stringio'
+
+require_relative '../core/utils/base64'
+
+module Datadog
+  module AppSec
+    # Converts derivative schema payloads into JSON and compresses them into a
+    # base64 encoded string if the payload is worth compressing.
+    #
+    # See: https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747221082
+    module CompressedJson
+      MIN_SIZE_FOR_COMPRESSION = 260
+
+      def self.dump(payload)
+        value = JSON.dump(payload)
+        return value if value.bytesize < MIN_SIZE_FOR_COMPRESSION
+
+        compress_and_encode(value)
+      rescue ArgumentError, Encoding::UndefinedConversionError, JSON::JSONError => e
+        AppSec.telemetry.report(e, description: 'AppSec: Failed to convert value into JSON')
+
+        nil
+      end
+
+      private_class_method def self.compress_and_encode(payload)
+        Core::Utils::Base64.strict_encode64(
+          Zlib.gzip(payload, level: Zlib::BEST_SPEED, strategy: Zlib::DEFAULT_STRATEGY)
+        )
+      rescue Zlib::Error, TypeError => e
+        AppSec.telemetry.report(e, description: 'AppSec: Failed to compress and encode value')
+
+        nil
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/active_record/integration.rb

@@ -13,7 +13,7 @@ module Datadog
 
           MINIMUM_VERSION = Gem::Version.new('4')
 
-          register_as :active_record, auto_patch: false
+          register_as :active_record, auto_patch: true
 
           def self.version
             Gem.loaded_specs['activerecord'] && Gem.loaded_specs['activerecord'].version

data/lib/datadog/appsec/event.rb

@@ -1,15 +1,15 @@
 # frozen_string_literal: true
 
 require 'json'
-require 'zlib'
-
 require_relative 'rate_limiter'
-require_relative '../core/utils/base64'
+require_relative 'compressed_json'
 
 module Datadog
   module AppSec
     # AppSec event
     module Event
+      DERIVATIVE_SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
+      DERIVATIVE_SCHEMA_MAX_COMPRESSED_SIZE = 25000
       ALLOWED_REQUEST_HEADERS = %w[
         X-Forwarded-For
         X-Client-IP
@@ -38,11 +38,6 @@ module Datadog
         Content-Language
       ].map!(&:downcase).freeze
 
-      MAX_ENCODED_SCHEMA_SIZE = 25000
-      # For more information about this number
-      # please check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747221082
-      MIN_SCHEMA_SIZE_FOR_COMPRESSION = 260
-
       # Record events for a trace
       #
       # This is expected to be called only once per trace for the rate limiter
@@ -80,7 +75,6 @@ module Datadog
           end
         end
 
-        # rubocop:disable Metrics/MethodLength
         def build_service_entry_tags(event_group)
           waf_events = []
           entry_tags = event_group.each_with_object({ '_dd.origin' => 'appsec' }) do |event, tags|
@@ -106,26 +100,17 @@ module Datadog
             waf_events += waf_result.events
 
             waf_result.derivatives.each do |key, value|
-              parsed_value = json_parse(value)
-              next unless parsed_value
-
-              parsed_value_size = parsed_value.size
-
-              schema_value = if parsed_value_size >= MIN_SCHEMA_SIZE_FOR_COMPRESSION
-                               compressed_and_base64_encoded(parsed_value)
-                             else
-                               parsed_value
-                             end
-              next unless schema_value
-
-              if schema_value.size >= MAX_ENCODED_SCHEMA_SIZE
-                Datadog.logger.debug do
-                  "Schema key: #{key} exceeds the max size value. It will not be included as part of the span tags"
-                end
+              next tags[key] = value unless key.start_with?(DERIVATIVE_SCHEMA_KEY_PREFIX)
+
+              value = CompressedJson.dump(value)
+              next if value.nil?
+
+              if value.size >= DERIVATIVE_SCHEMA_MAX_COMPRESSED_SIZE
+                Datadog.logger.debug { "AppSec: Schema key '#{key}' will not be included into span tags due to it's size" }
                 next
               end
 
-              tags[key] = schema_value
+              tags[key] = value
             end
 
             tags
@@ -135,14 +120,16 @@ module Datadog
           entry_tags['_dd.appsec.json'] = appsec_events if appsec_events
           entry_tags
         end
-        # rubocop:enable Metrics/MethodLength
 
         def tag_and_keep!(context, waf_result)
           # We want to keep the trace in case of security event
           context.trace.keep! if context.trace
 
           if context.span
-            context.span.set_tag('appsec.blocked', 'true') if waf_result.actions.key?('block_request')
+            if waf_result.actions.key?('block_request') || waf_result.actions.key?('redirect_request')
+              context.span.set_tag('appsec.blocked', 'true')
+            end
+
             context.span.set_tag('appsec.event', 'true')
           end
 
@@ -151,31 +138,15 @@ module Datadog
 
         private
 
-        def compressed_and_base64_encoded(value)
-          Datadog::Core::Utils::Base64.strict_encode64(gzip(value))
-        rescue TypeError => e
-          Datadog.logger.debug do
-            "Failed to compress and encode value when populating AppSec::Event. Error: #{e.message}"
-          end
-          nil
-        end
-
+        # NOTE: Handling of Encoding::UndefinedConversionError is added as a quick fix to
+        #       the issue between Ruby encoded strings and libddwaf produced events and now
+        #       is under investigation.
         def json_parse(value)
           JSON.dump(value)
-        rescue ArgumentError => e
-          Datadog.logger.debug do
-            "Failed to parse value to JSON when populating AppSec::Event. Error: #{e.message}"
-          end
-          nil
-        end
+        rescue ArgumentError, Encoding::UndefinedConversionError, JSON::JSONError => e
+          AppSec.telemetry.report(e, description: 'AppSec: Failed to convert value into JSON')
 
-        def gzip(value)
-          sio = StringIO.new
-          # For an in depth comparison of Zlib options check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747215473
-          gz = Zlib::GzipWriter.new(sio, Zlib::BEST_SPEED, Zlib::DEFAULT_STRATEGY)
-          gz.write(value)
-          gz.close
-          sio.string
+          nil
         end
 
         # Propagate to downstream services the information that the current distributed trace is

data/lib/datadog/appsec/remote.rb

@@ -23,6 +23,8 @@ module Datadog
         CAP_ASM_CUSTOM_RULES              = 1 << 8   # accept custom rules
         CAP_ASM_CUSTOM_BLOCKING_RESPONSE  = 1 << 9   # supports custom http code or redirect sa blocking response
         CAP_ASM_TRUSTED_IPS               = 1 << 10  # supports trusted ip
+        CAP_ASM_RASP_SSRF                 = 1 << 23  # support for server-side request forgery exploit prevention rules
+        CAP_ASM_RASP_SQLI                 = 1 << 21  # support for SQL injection exploit prevention rules
 
         # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
         ASM_CAPABILITIES = [
@@ -35,6 +37,8 @@ module Datadog
           CAP_ASM_CUSTOM_RULES,
           CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
           CAP_ASM_TRUSTED_IPS,
+          CAP_ASM_RASP_SSRF,
+          CAP_ASM_RASP_SQLI,
         ].freeze
 
         ASM_PRODUCTS = [

data/lib/datadog/core/diagnostics/environment_logger.rb

@@ -79,7 +79,7 @@ module Datadog
 
           # @return [String] current time in ISO8601 format
           def date
-            Time.now.utc.iso8601
+            Core::Utils::Time.now.utc.iso8601
           end
 
           # Best portable guess of OS information.

data/lib/datadog/core/environment/agent_info.rb

@@ -53,7 +53,7 @@ module Datadog
       class AgentInfo
         attr_reader :agent_settings, :logger
 
-        def initialize(agent_settings, logger:)
+        def initialize(agent_settings, logger: Datadog.logger)
           @agent_settings = agent_settings
           @logger = logger
           @client = Remote::Transport::HTTP.root(agent_settings: agent_settings, logger: logger)

data/lib/datadog/core/metrics/client.rb

@@ -23,7 +23,7 @@ module Datadog
 
         attr_reader :statsd, :logger
 
-        def initialize(logger:, statsd: nil, enabled: true, **_)
+        def initialize(logger: Datadog.logger, statsd: nil, enabled: true, **_)
           @logger = logger
           @statsd =
             if supported?

data/lib/datadog/core/remote/client.rb

@@ -15,7 +15,7 @@ module Datadog
 
         attr_reader :transport, :repository, :id, :dispatcher, :logger
 
-        def initialize(transport, capabilities, logger:, repository: Configuration::Repository.new)
+        def initialize(transport, capabilities, logger: Datadog.logger, repository: Configuration::Repository.new)
           @transport = transport
           @logger = logger
 

data/lib/datadog/core/remote/negotiation.rb

@@ -9,7 +9,7 @@ module Datadog
       class Negotiation
         attr_reader :logger
 
-        def initialize(_settings, agent_settings, logger:, suppress_logging: {})
+        def initialize(_settings, agent_settings, logger: Datadog.logger, suppress_logging: {})
           @logger = logger
           @transport_root = Datadog::Core::Remote::Transport::HTTP.root(agent_settings: agent_settings, logger: logger)
           @logged = suppress_logging

data/lib/datadog/core/remote/transport/config.rb

@@ -34,11 +34,11 @@ module Datadog
           class Transport
             attr_reader :client, :apis, :default_api, :current_api_id, :logger
 
-            def initialize(apis, default_api, logger)
+            def initialize(apis, default_api, logger: Datadog.logger)
               @apis = apis
               @logger = logger
 
-              @client = HTTP::Client.new(current_api, logger)
+              @client = HTTP::Client.new(current_api, logger: logger)
             end
 
             ##### there is only one transport! it's negotiation!

data/lib/datadog/core/remote/transport/http/client.rb

@@ -17,7 +17,7 @@ module Datadog
           class Client
             attr_reader :api, :logger
 
-            def initialize(api, logger)
+            def initialize(api, logger: Datadog.logger)
               @api = api
               @logger = logger
             end

data/lib/datadog/core/remote/transport/http.rb

@@ -33,7 +33,7 @@ module Datadog
           # Pass a block to override any settings.
           def root(
             agent_settings:,
-            logger:,
+            logger: Datadog.logger,
             api_version: nil,
             headers: nil
           )
@@ -57,7 +57,7 @@ module Datadog
           # Pass a block to override any settings.
           def v7(
             agent_settings:,
-            logger:,
+            logger: Datadog.logger,
             api_version: nil,
             headers: nil
           )

data/lib/datadog/core/remote/transport/negotiation.rb

@@ -51,11 +51,11 @@ module Datadog
           class Transport
             attr_reader :client, :apis, :default_api, :current_api_id, :logger
 
-            def initialize(apis, default_api, logger)
+            def initialize(apis, default_api, logger: Datadog.logger)
               @apis = apis
               @logger = logger
 
-              @client = HTTP::Client.new(current_api, logger)
+              @client = HTTP::Client.new(current_api, logger: logger)
             end
 
             def send_info

data/lib/datadog/core/telemetry/metric.rb

@@ -108,9 +108,9 @@ module Datadog
             value = value.to_i
 
             if values.empty?
-              values << [Time.now.to_i, value]
+              values << [Core::Utils::Time.now.to_i, value]
             else
-              values[0][0] = Time.now.to_i
+              values[0][0] = Core::Utils::Time.now.to_i
               values[0][1] += value
             end
             nil
@@ -129,9 +129,9 @@ module Datadog
 
           def track(value)
             if values.empty?
-              values << [Time.now.to_i, value]
+              values << [Core::Utils::Time.now.to_i, value]
             else
-              values[0][0] = Time.now.to_i
+              values[0][0] = Core::Utils::Time.now.to_i
               values[0][1] = value
             end
             nil
@@ -155,7 +155,7 @@ module Datadog
 
           def track(value = 1.0)
             @value += value
-            @values = [[Time.now.to_i, @value / interval]]
+            @values = [[Core::Utils::Time.now.to_i, @value / interval]]
             nil
           end
         end

data/lib/datadog/core/telemetry/request.rb

@@ -21,7 +21,7 @@ module Datadog
               request_type: event.type,
               runtime_id: Core::Environment::Identity.id,
               seq_id: seq_id,
-              tracer_time: Time.now.to_i,
+              tracer_time: Core::Utils::Time.now.to_i,
             }
             hash.compact!
             hash

data/lib/datadog/core/transport/http/builder.rb

@@ -21,7 +21,7 @@ module Datadog
             :default_headers,
             :logger
 
-          def initialize(api_instance_class:, logger:)
+          def initialize(api_instance_class:, logger: Datadog.logger)
             # Global settings
             @default_adapter = nil
             @default_headers = {}
@@ -88,7 +88,7 @@ module Datadog
           def to_transport(klass)
             raise NoDefaultApiError if @default_api.nil?
 
-            klass.new(to_api_instances, @default_api, logger)
+            klass.new(to_api_instances, @default_api, logger: logger)
           end
 
           def to_api_instances

data/lib/datadog/core/transport/http.rb

@@ -29,7 +29,7 @@ module Datadog
         # Helper function that delegates to Builder.new
         # but is under HTTP namespace so that client code requires this file
         # to get the adapters configured, and not the builder directly.
-        def build(api_instance_class:, agent_settings:, logger:, api_version: nil, headers: nil, &block)
+        def build(api_instance_class:, agent_settings:, logger: Datadog.logger, api_version: nil, headers: nil, &block)
           Builder.new(api_instance_class: api_instance_class, logger: logger) do |transport|
             transport.adapter(agent_settings)
             transport.headers(default_headers)

data/lib/datadog/di/probe_notification_builder.rb

@@ -191,7 +191,7 @@ module Datadog
       end
 
       def timestamp_now
-        (Time.now.to_f * 1000).to_i
+        (Core::Utils::Time.now.to_f * 1000).to_i
       end
 
       def get_local_variables(trace_point)

data/lib/datadog/di/transport/diagnostics.rb

@@ -17,11 +17,11 @@ module Datadog
         class Transport
           attr_reader :client, :apis, :default_api, :current_api_id, :logger
 
-          def initialize(apis, default_api, logger)
+          def initialize(apis, default_api, logger:)
             @apis = apis
             @logger = logger
 
-            @client = HTTP::Client.new(current_api, logger)
+            @client = HTTP::Client.new(current_api, logger: logger)
           end
 
           def current_api

data/lib/datadog/di/transport/http/client.rb

@@ -16,7 +16,7 @@ module Datadog
         class Client
           attr_reader :api, :logger
 
-          def initialize(api, logger)
+          def initialize(api, logger:)
             @api = api
             @logger = logger
           end

data/lib/datadog/di/transport/http/diagnostics.rb

@@ -38,7 +38,6 @@ module Datadog
               end
             end
 
-            # Endpoint for negotiation
             class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
               attr_reader :encoder
 

data/lib/datadog/di/transport/http/input.rb

@@ -38,7 +38,6 @@ module Datadog
               end
             end
 
-            # Endpoint for negotiation
             class Endpoint < Datadog::Core::Transport::HTTP::API::Endpoint
               HEADER_CONTENT_TYPE = 'Content-Type'
 

data/lib/datadog/di/transport/http.rb

@@ -1,15 +1,9 @@
 # frozen_string_literal: true
 
-require 'uri'
-
-require_relative '../../core/environment/container'
-require_relative '../../core/environment/ext'
-require_relative '../../core/transport/ext'
 require_relative 'diagnostics'
 require_relative 'input'
 require_relative 'http/api'
 require_relative '../../core/transport/http'
-require_relative '../../../datadog/version'
 
 module Datadog
   module DI

data/lib/datadog/di/transport/input.rb

@@ -17,11 +17,11 @@ module Datadog
         class Transport
           attr_reader :client, :apis, :default_api, :current_api_id, :logger
 
-          def initialize(apis, default_api, logger)
+          def initialize(apis, default_api, logger:)
             @apis = apis
             @logger = logger
 
-            @client = HTTP::Client.new(current_api, logger)
+            @client = HTTP::Client.new(current_api, logger: logger)
           end
 
           def current_api

data/lib/datadog/kit/appsec/events.rb

@@ -10,6 +10,7 @@ module Datadog
         LOGIN_SUCCESS_EVENT = 'users.login.success'
         LOGIN_FAILURE_EVENT = 'users.login.failure'
         SIGNUP_EVENT = 'users.signup'
+        USER_LOGIN_KEYS = ['usr.login', :'usr.login'].freeze
 
         class << self
           # Attach login success event information to the trace
@@ -30,10 +31,11 @@ module Datadog
             set_trace_and_span_context('track_login_success', trace, span) do |active_trace, active_span|
               user_options = user.dup
               user_id = user_options.delete(:id)
-              user_login = user_options[:login] || others[:'usr.login'] || user_id
+              user_login = user_options[:login] || others[:'usr.login'] || others['usr.login'] || user_id
 
               raise ArgumentError, 'missing required key: :user => { :id }' if user_id.nil?
 
+              others = others.reject { |key, _| USER_LOGIN_KEYS.include?(key) }
               others[:'usr.login'] = user_login
               track(LOGIN_SUCCESS_EVENT, active_trace, active_span, **others)
 
@@ -58,7 +60,7 @@ module Datadog
           #   event information to attach to the trace.
           def track_login_failure(trace = nil, span = nil, user_exists:, user_id: nil, **others)
             set_trace_and_span_context('track_login_failure', trace, span) do |active_trace, active_span|
-              others[:'usr.login'] = user_id if user_id && !others.key?(:'usr.login')
+              others[:'usr.login'] = user_id if user_id && !others.key?(:'usr.login') && !others.key?('usr.login')
               track(LOGIN_FAILURE_EVENT, active_trace, active_span, **others)
 
               active_span.set_tag('appsec.events.users.login.failure.usr.id', user_id) if user_id
@@ -84,10 +86,11 @@ module Datadog
             set_trace_and_span_context('track_signup', trace, span) do |active_trace, active_span|
               user_options = user.dup
               user_id = user_options.delete(:id)
-              user_login = user_options[:login] || others[:'usr.login'] || user_id
+              user_login = user_options[:login] || others[:'usr.login'] || others['usr.login'] || user_id
 
               raise ArgumentError, 'missing required key: :user => { :id }' if user_id.nil?
 
+              others = others.reject { |key, _| USER_LOGIN_KEYS.include?(key) }
               others[:'usr.login'] = user_login
               track(SIGNUP_EVENT, active_trace, active_span, **others)
 

data/lib/datadog/profiling/collectors/info.rb

@@ -31,6 +31,9 @@ module Datadog
         # Instead of trying to figure out real process start time by checking
         # /proc or some other complex/non-portable way, approximate start time
         # by time of requirement of this file.
+        #
+        # Note: this does not use Core::Utils::Time.now because this constant
+        # gets initialized before a user has a chance to configure the library.
         START_TIME = Time.now.utc.freeze
 
         def collect_platform_info

data/lib/datadog/profiling/encoded_profile.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Profiling
+    # This class exists to wrap a ddog_prof_EncodedProfile into a Ruby object
+    #
+    # This class is not empty; all of this class is implemented as native code.
+    class EncodedProfile # rubocop:disable Lint/EmptyClass
+    end
+  end
+end

data/lib/datadog/profiling/exporter.rb

@@ -57,7 +57,7 @@ module Datadog
         serialization_result = pprof_recorder.serialize
         return if serialization_result.nil?
 
-        start, finish, compressed_pprof, profile_stats = serialization_result
+        start, finish, encoded_profile, profile_stats = serialization_result
         @last_flush_finish_at = finish
 
         if duration_below_threshold?(start, finish)
@@ -70,8 +70,7 @@ module Datadog
         Flush.new(
           start: start,
           finish: finish,
-          pprof_file_name: Datadog::Profiling::Ext::Transport::HTTP::PPROF_DEFAULT_FILENAME,
-          pprof_data: compressed_pprof.to_s,
+          encoded_profile: encoded_profile,
           code_provenance_file_name: Datadog::Profiling::Ext::Transport::HTTP::CODE_PROVENANCE_FILENAME,
           code_provenance_data: uncompressed_code_provenance,
           tags_as_array: Datadog::Profiling::TagBuilder.call(settings: Datadog.configuration).to_a,

data/lib/datadog/profiling/ext.rb

@@ -26,7 +26,6 @@ module Datadog
           TAG_GIT_REPOSITORY_URL = "git.repository_url"
           TAG_GIT_COMMIT_SHA = "git.commit.sha"
 
-          PPROF_DEFAULT_FILENAME = "rubyprofile.pprof"
           CODE_PROVENANCE_FILENAME = "code-provenance.json"
         end
       end

data/lib/datadog/profiling/flush.rb

@@ -9,10 +9,9 @@ module Datadog
       attr_reader \
         :start,
         :finish,
-        :pprof_file_name,
-        :pprof_data, # gzipped pprof bytes
+        :encoded_profile,
         :code_provenance_file_name,
-        :code_provenance_data, # gzipped json bytes
+        :code_provenance_data,
         :tags_as_array,
         :internal_metadata_json,
         :info_json
@@ -20,8 +19,7 @@ module Datadog
       def initialize(
         start:,
         finish:,
-        pprof_file_name:,
-        pprof_data:,
+        encoded_profile:,
         code_provenance_file_name:,
         code_provenance_data:,
         tags_as_array:,
@@ -30,8 +28,7 @@ module Datadog
       )
         @start = start
         @finish = finish
-        @pprof_file_name = pprof_file_name
-        @pprof_data = pprof_data
+        @encoded_profile = encoded_profile
         @code_provenance_file_name = code_provenance_file_name
         @code_provenance_data = code_provenance_data
         @tags_as_array = tags_as_array