datadog 2.12.2 → 2.15.0

data/lib/datadog/appsec/contrib/devise/tracking_middleware.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require_relative 'ext'
+require_relative '../../anonymizer'
+
+module Datadog
+  module AppSec
+    module Contrib
+      module Devise
+        # A Rack middleware capable of tracking currently signed user
+        class TrackingMiddleware
+          WARDEN_KEY = 'warden'
+
+          def initialize(app)
+            @app = app
+            @devise_session_scope_keys = {}
+          end
+
+          def call(env)
+            return @app.call(env) unless AppSec.enabled?
+            return @app.call(env) unless Configuration.auto_user_instrumentation_enabled?
+            return @app.call(env) unless AppSec.active_context
+
+            unless env.key?(WARDEN_KEY)
+              Datadog.logger.debug { 'AppSec: unable to track requests, due to missing warden manager' }
+              return @app.call(env)
+            end
+
+            context = AppSec.active_context
+            if context.trace.nil? || context.span.nil?
+              Datadog.logger.debug { 'AppSec: unable to track requests, due to missing trace or span' }
+              return @app.call(env)
+            end
+
+            id = transform(extract_id(env[WARDEN_KEY]))
+            if id
+              unless context.span.has_tag?(Ext::TAG_USR_ID)
+                context.span[Ext::TAG_USR_ID] = id
+                AppSec::Instrumentation.gateway.push(
+                  'identity.set_user', AppSec::Instrumentation::Gateway::User.new(id, nil)
+                )
+              end
+
+              context.span[Ext::TAG_DD_USR_ID] = id.to_s
+              context.span[Ext::TAG_DD_COLLECTION_MODE] ||= Configuration.auto_user_instrumentation_mode
+            end
+
+            @app.call(env)
+          end
+
+          private
+
+          def extract_id(warden)
+            session_serializer = warden.session_serializer
+
+            key = session_key_for(session_serializer, ::Devise.default_scope)
+            id = session_serializer.session[key]&.dig(0, 0)
+
+            return id if ::Devise.mappings.size == 1
+            return "#{::Devise.default_scope}:#{id}" if id
+
+            ::Devise.mappings.each_key do |scope|
+              next if scope == ::Devise.default_scope
+
+              key = session_key_for(session_serializer, scope)
+              id = session_serializer.session[key]&.dig(0, 0)
+
+              return "#{scope}:#{id}" if id
+            end
+
+            nil
+          end
+
+          def session_key_for(session_serializer, scope)
+            @devise_session_scope_keys[scope] ||= session_serializer.key_for(scope)
+          end
+
+          def transform(value)
+            return if value.nil?
+            return value.to_s unless anonymize?
+
+            Anonymizer.anonimyze(value.to_s)
+          end
+
+          def anonymize?
+            Configuration.auto_user_instrumentation_mode ==
+              AppSec::Configuration::Settings::ANONYMIZATION_AUTO_USER_INSTRUMENTATION_MODE
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/contrib/rack/ext.rb

@@ -6,6 +6,20 @@ module Datadog
       module Rack
         # Rack integration constants
         module Ext
+          COLLECTABLE_REQUEST_HEADERS = [
+            'accept',
+            'akamai-user-risk',
+            'cf-ray',
+            'cloudfront-viewer-ja3-fingerprint',
+            'content-type',
+            'user-agent',
+            'x-amzn-trace-Id',
+            'x-appgw-trace-id',
+            'x-cloud-trace-context',
+            'x-sigsci-requestid',
+            'x-sigsci-tags'
+          ].freeze
+
           IDENTITY_COLLECTABLE_REQUEST_HEADERS = [
             'accept-encoding',
             'accept-language',

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -18,6 +18,7 @@ module Datadog
                 watch_request(gateway)
                 watch_response(gateway)
                 watch_request_body(gateway)
+                watch_request_finish(gateway)
               end
 
               def watch_request(gateway = Instrumentation.gateway)
@@ -119,12 +120,18 @@ module Datadog
               def watch_request_finish(gateway = Instrumentation.gateway)
                 gateway.watch('rack.request.finish', :appsec) do |stack, gateway_request|
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
-                  next stack.call(gateway_request.request) if context.span.nil? || !gateway.pushed?('identity.set_user')
+
+                  if context.span.nil? || !gateway.pushed?('appsec.events.user_lifecycle')
+                    next stack.call(gateway_request.request)
+                  end
 
                   gateway_request.headers.each do |name, value|
-                    next unless Ext::IDENTITY_COLLECTABLE_REQUEST_HEADERS.include?(name)
+                    if !Ext::COLLECTABLE_REQUEST_HEADERS.include?(name) &&
+                        !Ext::IDENTITY_COLLECTABLE_REQUEST_HEADERS.include?(name)
+                      next
+                    end
 
-                    context.span["http.request.headers.#{name}"] = value
+                    context.span["http.request.headers.#{name}"] ||= value
                   end
 
                   stack.call(gateway_request.request)

data/lib/datadog/appsec/contrib/rack/request_middleware.rb

@@ -150,8 +150,6 @@ module Datadog
             return unless trace && span
 
             span.set_metric(Datadog::AppSec::Ext::TAG_APPSEC_ENABLED, 1)
-            # We add this tag when ASM standalone is enabled to make sure we don't bill APM
-            span.set_metric(Datadog::AppSec::Ext::TAG_APM_ENABLED, 0) if Datadog.configuration.appsec.standalone.enabled
             span.set_tag('_dd.runtime_family', 'ruby')
             span.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
 

data/lib/datadog/appsec/event.rb

@@ -1,15 +1,15 @@
 # frozen_string_literal: true
 
 require 'json'
-require 'zlib'
-
 require_relative 'rate_limiter'
-require_relative '../core/utils/base64'
+require_relative 'compressed_json'
 
 module Datadog
   module AppSec
     # AppSec event
     module Event
+      DERIVATIVE_SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
+      DERIVATIVE_SCHEMA_MAX_COMPRESSED_SIZE = 25000
       ALLOWED_REQUEST_HEADERS = %w[
         X-Forwarded-For
         X-Client-IP
@@ -38,11 +38,6 @@ module Datadog
         Content-Language
       ].map!(&:downcase).freeze
 
-      MAX_ENCODED_SCHEMA_SIZE = 25000
-      # For more information about this number
-      # please check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747221082
-      MIN_SCHEMA_SIZE_FOR_COMPRESSION = 260
-
       # Record events for a trace
       #
       # This is expected to be called only once per trace for the rate limiter
@@ -80,7 +75,6 @@ module Datadog
           end
         end
 
-        # rubocop:disable Metrics/MethodLength
         def build_service_entry_tags(event_group)
           waf_events = []
           entry_tags = event_group.each_with_object({ '_dd.origin' => 'appsec' }) do |event, tags|
@@ -106,26 +100,17 @@ module Datadog
             waf_events += waf_result.events
 
             waf_result.derivatives.each do |key, value|
-              parsed_value = json_parse(value)
-              next unless parsed_value
-
-              parsed_value_size = parsed_value.size
-
-              schema_value = if parsed_value_size >= MIN_SCHEMA_SIZE_FOR_COMPRESSION
-                               compressed_and_base64_encoded(parsed_value)
-                             else
-                               parsed_value
-                             end
-              next unless schema_value
-
-              if schema_value.size >= MAX_ENCODED_SCHEMA_SIZE
-                Datadog.logger.debug do
-                  "Schema key: #{key} exceeds the max size value. It will not be included as part of the span tags"
-                end
+              next tags[key] = value unless key.start_with?(DERIVATIVE_SCHEMA_KEY_PREFIX)
+
+              value = CompressedJson.dump(value)
+              next if value.nil?
+
+              if value.size >= DERIVATIVE_SCHEMA_MAX_COMPRESSED_SIZE
+                Datadog.logger.debug { "AppSec: Schema key '#{key}' will not be included into span tags due to it's size" }
                 next
               end
 
-              tags[key] = schema_value
+              tags[key] = value
             end
 
             tags
@@ -135,14 +120,16 @@ module Datadog
           entry_tags['_dd.appsec.json'] = appsec_events if appsec_events
           entry_tags
         end
-        # rubocop:enable Metrics/MethodLength
 
         def tag_and_keep!(context, waf_result)
           # We want to keep the trace in case of security event
           context.trace.keep! if context.trace
 
           if context.span
-            context.span.set_tag('appsec.blocked', 'true') if waf_result.actions.key?('block_request')
+            if waf_result.actions.key?('block_request') || waf_result.actions.key?('redirect_request')
+              context.span.set_tag('appsec.blocked', 'true')
+            end
+
             context.span.set_tag('appsec.event', 'true')
           end
 
@@ -151,31 +138,15 @@ module Datadog
 
         private
 
-        def compressed_and_base64_encoded(value)
-          Datadog::Core::Utils::Base64.strict_encode64(gzip(value))
-        rescue TypeError => e
-          Datadog.logger.debug do
-            "Failed to compress and encode value when populating AppSec::Event. Error: #{e.message}"
-          end
-          nil
-        end
-
+        # NOTE: Handling of Encoding::UndefinedConversionError is added as a quick fix to
+        #       the issue between Ruby encoded strings and libddwaf produced events and now
+        #       is under investigation.
         def json_parse(value)
           JSON.dump(value)
-        rescue ArgumentError => e
-          Datadog.logger.debug do
-            "Failed to parse value to JSON when populating AppSec::Event. Error: #{e.message}"
-          end
-          nil
-        end
+        rescue ArgumentError, Encoding::UndefinedConversionError, JSON::JSONError => e
+          AppSec.telemetry.report(e, description: 'AppSec: Failed to convert value into JSON')
 
-        def gzip(value)
-          sio = StringIO.new
-          # For an in depth comparison of Zlib options check https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747215473
-          gz = Zlib::GzipWriter.new(sio, Zlib::BEST_SPEED, Zlib::DEFAULT_STRATEGY)
-          gz.write(value)
-          gz.close
-          sio.string
+          nil
         end
 
         # Propagate to downstream services the information that the current distributed trace is
@@ -187,7 +158,7 @@ module Datadog
             Datadog::Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
             Datadog::Tracing::Sampling::Ext::Decision::ASM
           )
-          trace.set_tag(Datadog::AppSec::Ext::TAG_DISTRIBUTED_APPSEC_EVENT, '1')
+          trace.set_distributed_source(Datadog::AppSec::Ext::PRODUCT_BIT)
         end
       end
     end

data/lib/datadog/appsec/ext.rb

@@ -7,13 +7,15 @@ module Datadog
       RASP_LFI = 'lfi'
       RASP_SSRF = 'ssrf'
 
+      PRODUCT_BIT = 0b00000010
+
       INTERRUPT = :datadog_appsec_interrupt
       CONTEXT_KEY = 'datadog.appsec.context'
       ACTIVE_CONTEXT_KEY = :datadog_appsec_active_context
+      EXPLOIT_PREVENTION_EVENT_CATEGORY = 'exploit'
 
       TAG_APPSEC_ENABLED = '_dd.appsec.enabled'
-      TAG_APM_ENABLED = '_dd.apm.enabled'
-      TAG_DISTRIBUTED_APPSEC_EVENT = '_dd.p.appsec'
+      TAG_METASTRUCT_STACK_TRACE = '_dd.stack'
 
       TELEMETRY_METRICS_NAMESPACE = 'appsec'
     end

data/lib/datadog/appsec/instrumentation/gateway/argument.rb

@@ -9,11 +9,13 @@ module Datadog
 
         # Gateway User argument
         class User < Argument
-          attr_reader :id
+          attr_reader :id, :login
 
-          def initialize(id)
+          def initialize(id, login)
             super()
+
             @id = id
+            @login = login
           end
         end
       end

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -19,9 +19,14 @@ module Datadog
               gateway.watch('identity.set_user', :appsec) do |stack, user|
                 context = Datadog::AppSec.active_context
 
-                persistent_data = {
-                  'usr.id' => user.id
-                }
+                if user.id.nil? && user.login.nil?
+                  Datadog.logger.debug { 'AppSec: skipping WAF check because no user information was provided' }
+                  next stack.call(user)
+                end
+
+                persistent_data = {}
+                persistent_data['usr.id'] = user.id if user.id
+                persistent_data['usr.login'] = user.login if user.login
 
                 result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 

data/lib/datadog/appsec/remote.rb

@@ -23,6 +23,8 @@ module Datadog
         CAP_ASM_CUSTOM_RULES              = 1 << 8   # accept custom rules
         CAP_ASM_CUSTOM_BLOCKING_RESPONSE  = 1 << 9   # supports custom http code or redirect sa blocking response
         CAP_ASM_TRUSTED_IPS               = 1 << 10  # supports trusted ip
+        CAP_ASM_RASP_SSRF                 = 1 << 23  # support for server-side request forgery exploit prevention rules
+        CAP_ASM_RASP_SQLI                 = 1 << 21  # support for SQL injection exploit prevention rules
 
         # TODO: we need to dynamically add CAP_ASM_ACTIVATION once we support it
         ASM_CAPABILITIES = [
@@ -35,6 +37,8 @@ module Datadog
           CAP_ASM_CUSTOM_RULES,
           CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
           CAP_ASM_TRUSTED_IPS,
+          CAP_ASM_RASP_SSRF,
+          CAP_ASM_RASP_SQLI,
         ].freeze
 
         ASM_PRODUCTS = [

data/lib/datadog/appsec/security_engine/runner.rb

@@ -24,13 +24,13 @@ module Datadog
           persistent_data.reject! do |_, v|
             next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
 
-            v.nil? ? true : v.empty?
+            v.nil? || v.empty?
           end
 
           ephemeral_data.reject! do |_, v|
             next false if v.is_a?(TrueClass) || v.is_a?(FalseClass)
 
-            v.nil? ? true : v.empty?
+            v.nil? || v.empty?
           end
 
           _code, result = try_run(persistent_data, ephemeral_data, timeout)

data/lib/datadog/appsec/utils.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require_relative 'utils/trace_operation'
-
 module Datadog
   module AppSec
     # Utilities for AppSec

data/lib/datadog/core/configuration/components.rb

@@ -45,6 +45,7 @@ module Datadog
             options = { enabled: settings.runtime_metrics.enabled }
             options[:statsd] = settings.runtime_metrics.statsd unless settings.runtime_metrics.statsd.nil?
             options[:services] = [settings.service] unless settings.service.nil?
+            options[:experimental_runtime_id_enabled] = settings.runtime_metrics.experimental_runtime_id_enabled
 
             Core::Runtime::Metrics.new(logger: logger, **options)
           end
@@ -101,7 +102,7 @@ module Datadog
           agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
 
           # Exposes agent capability information for detection by any components
-          @agent_info = Core::Environment::AgentInfo.new(agent_settings)
+          @agent_info = Core::Environment::AgentInfo.new(agent_settings, logger: @logger)
 
           @telemetry = self.class.build_telemetry(settings, agent_settings, @logger)
 

data/lib/datadog/core/configuration/ext.rb

@@ -18,6 +18,10 @@ module Datadog
           ENV_DEFAULT_PORT = 'DD_METRIC_AGENT_PORT'
         end
 
+        module APM
+          ENV_TRACING_ENABLED = 'DD_APM_TRACING_ENABLED'
+        end
+
         module Agent
           ENV_DEFAULT_HOST = 'DD_AGENT_HOST'
           # Some env vars have "trace" in them, but they apply to all products

data/lib/datadog/core/configuration/options.rb

@@ -45,7 +45,7 @@ module Datadog
               option_name.to_sym => proc do
                 get_option(option_name)
               end,
-              "#{option_name}=".to_sym => proc do |value|
+              :"#{option_name}=" => proc do |value|
                 set_option(option_name, value)
               end
             }
@@ -117,7 +117,7 @@ module Datadog
           end
 
           def resolved_env(name)
-            return options[name].resolved_env if options.key?(name)
+            options[name].resolved_env if options.key?(name)
           end
 
           def assert_valid_option!(name)

data/lib/datadog/core/configuration/settings.rb

@@ -570,6 +570,12 @@ module Datadog
             o.type :bool
           end
 
+          option :experimental_runtime_id_enabled do |o|
+            o.type :bool
+            o.env 'DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED'
+            o.default false
+          end
+
           option :opts, default: {}, type: :hash
           option :statsd
         end
@@ -619,38 +625,31 @@ module Datadog
           o.type :hash, nilable: true
           o.env [Core::Environment::Ext::ENV_TAGS, Core::Environment::Ext::ENV_OTEL_RESOURCE_ATTRIBUTES]
           o.env_parser do |env_value|
-            values = if env_value.include?(',')
-                       env_value.split(',')
-                     else
-                       env_value.split(' ') # rubocop:disable Style/RedundantArgument
-                     end
-            values.map! do |v|
-              v.gsub!(/\A[\s,]*|[\s,]*\Z/, '')
-
-              v.empty? ? nil : v
-            end
-
-            values.compact!
-            values.each_with_object({}) do |tag, tags|
-              key, value = tag.split(':', 2)
-              if value.nil?
-                # support tags/attributes delimited by the OpenTelemetry separator (`=`)
-                key, value = tag.split('=', 2)
-              end
-              next if value.nil? || value.empty?
-
-              # maps OpenTelemetry semantic attributes to Datadog tags
-              case key.downcase
-              when 'deployment.environment'
-                tags['env'] = value
-              when 'service.version'
-                tags['version'] = value
-              when 'service.name'
-                tags['service'] = value
-              else
-                tags[key] = value
+            # Parses a string containing key-value pairs and returns a hash.
+            # Key-value pairs are delimited by ':' OR `=`, and pairs are separated by whitespace, comma, OR BOTH.
+            result = {}
+            unless env_value.nil? || env_value.empty?
+              # falling back to comma as separator
+              sep = env_value.include?(',') ? ',' : ' '
+              # split by separator
+              env_value.split(sep).each do |tag|
+                tag.strip!
+                next if tag.empty?
+
+                # tag by : or = (for OpenTelemetry)
+                key, val = tag.split(/[:=]/, 2).map(&:strip)
+                val ||= ''
+                # maps OpenTelemetry semantic attributes to Datadog tags
+                key = case key.downcase
+                      when 'deployment.environment' then 'env'
+                      when 'service.version' then 'version'
+                      when 'service.name' then 'service'
+                      else key
+                      end
+                result[key] = val unless key.empty?
               end
             end
+            result
           end
           o.setter do |new_value, old_value|
             raw_tags = new_value || {}
@@ -958,6 +957,30 @@ module Datadog
           end
         end
 
+        # Tracer specific configuration starting with APM (e.g. DD_APM_TRACING_ENABLED).
+        # @public_api
+        settings :apm do
+          # Tracing as a transport
+          # @public_api
+          settings :tracing do
+            # Enables tracing as transport.
+            # Disabling it will set sampling priority to -1 (FORCE_DROP) on most traces,
+            # (which tells to the agent to drop these traces)
+            # except heartbeat ones (1 per minute) and manually kept ones (sampling priority to 2) (e.g. appsec events)
+            #
+            # This is different than `DD_TRACE_ENABLED`, which completely disables tracing (sends no trace at all),
+            # while this will send heartbeat traces (1 per minute) so that the service is considered alive in the backend.
+            #
+            # @default `DD_APM_TRACING_ENABLED` environment variable, otherwise `true`
+            # @return [Boolean]
+            option :enabled do |o|
+              o.env Configuration::Ext::APM::ENV_TRACING_ENABLED
+              o.default true
+              o.type :bool
+            end
+          end
+        end
+
         # TODO: Tracing should manage its own settings.
         #       Keep this extension here for now to keep things working.
         extend Datadog::Tracing::Configuration::Settings

data/lib/datadog/core/diagnostics/environment_logger.rb

@@ -79,7 +79,7 @@ module Datadog
 
           # @return [String] current time in ISO8601 format
           def date
-            Time.now.utc.iso8601
+            Core::Utils::Time.now.utc.iso8601
           end
 
           # Best portable guess of OS information.

data/lib/datadog/core/environment/agent_info.rb

@@ -51,11 +51,12 @@ module Datadog
       #
       # @see https://github.com/DataDog/datadog-agent/blob/f07df0a3c1fca0c83b5a15f553bd994091b0c8ac/pkg/trace/api/info.go#L20
       class AgentInfo
-        attr_reader :agent_settings
+        attr_reader :agent_settings, :logger
 
-        def initialize(agent_settings)
+        def initialize(agent_settings, logger: Datadog.logger)
           @agent_settings = agent_settings
-          @client = Remote::Transport::HTTP.root(agent_settings: agent_settings)
+          @logger = logger
+          @client = Remote::Transport::HTTP.root(agent_settings: agent_settings, logger: logger)
         end
 
         # Fetches the information from the agent.

data/lib/datadog/core/metrics/client.rb

@@ -23,7 +23,7 @@ module Datadog
 
         attr_reader :statsd, :logger
 
-        def initialize(logger:, statsd: nil, enabled: true, **_)
+        def initialize(logger: Datadog.logger, statsd: nil, enabled: true, **_)
           @logger = logger
           @statsd =
             if supported?

data/lib/datadog/core/remote/client.rb

@@ -15,7 +15,7 @@ module Datadog
 
         attr_reader :transport, :repository, :id, :dispatcher, :logger
 
-        def initialize(transport, capabilities, logger:, repository: Configuration::Repository.new)
+        def initialize(transport, capabilities, logger: Datadog.logger, repository: Configuration::Repository.new)
           @transport = transport
           @logger = logger
 

data/lib/datadog/core/remote/component.rb

@@ -18,11 +18,8 @@ module Datadog
         def initialize(settings, capabilities, agent_settings, logger:)
           @logger = logger
 
-          transport_options = {}
-          transport_options[:agent_settings] = agent_settings if agent_settings
-
-          negotiation = Negotiation.new(settings, agent_settings)
-          transport_v7 = Datadog::Core::Remote::Transport::HTTP.v7(**transport_options) # steep:ignore
+          negotiation = Negotiation.new(settings, agent_settings, logger: logger)
+          transport_v7 = Datadog::Core::Remote::Transport::HTTP.v7(agent_settings: agent_settings, logger: logger)
 
           @barrier = Barrier.new(settings.remote.boot_timeout_seconds)
 
@@ -49,7 +46,7 @@ module Datadog
               # In case of unexpected errors, reset the negotiation object
               # given external conditions have changed and the negotiation
               # negotiation object stores error logging state that should be reset.
-              negotiation = Negotiation.new(settings, agent_settings)
+              negotiation = Negotiation.new(settings, agent_settings, logger: logger)
 
               # Transient errors due to network or agent. Logged the error but not via telemetry
               logger.error do

data/lib/datadog/core/remote/configuration/repository.rb

@@ -272,7 +272,8 @@ module Datadog
 
               return deleted(path, previous) if previous && content.nil?
               return inserted(path, content) if content && previous.nil?
-              return updated(path, content, previous) if content && previous
+
+              updated(path, content, previous) if content && previous
             end
 
             def deleted(path, previous)