ddtrace 1.14.0 → 1.16.2

data/lib/datadog/appsec/monitor/gateway/watcher.rb

@@ -24,7 +24,7 @@ module Datadog
                 scope = Datadog::AppSec.active_scope
 
                 AppSec::Reactive::Operation.new('identity.set_user') do |op|
-                  Monitor::Reactive::SetUser.subscribe(op, scope.processor_context) do |result, _block|
+                  Monitor::Reactive::SetUser.subscribe(op, scope.processor_context) do |result|
                     if result.status == :match
                       # TODO: should this hash be an Event instance instead?
                       event = {
@@ -44,10 +44,10 @@ module Datadog
                     end
                   end
 
-                  _result, block = Monitor::Reactive::SetUser.publish(op, user)
+                  block = Monitor::Reactive::SetUser.publish(op, user)
                 end
 
-                throw(Datadog::AppSec::Ext::INTERRUPT, [nil, [:block, event]]) if block
+                throw(Datadog::AppSec::Ext::INTERRUPT, [nil, [[:block, event]]]) if block
 
                 ret, res = stack.call(user)
 

data/lib/datadog/appsec/monitor/reactive/set_user.rb

@@ -38,11 +38,8 @@ module Datadog
               when :match
                 Datadog.logger.debug { "WAF: #{result.inspect}" }
 
-                block = result.actions.include?('block')
-
-                yield [result, block]
-
-                throw(:block, [result, true]) if block
+                yield result
+                throw(:block, true) unless result.actions.empty?
               when :ok
                 Datadog.logger.debug { "WAF OK: #{result.inspect}" }
               when :invalid_call

data/lib/datadog/appsec/processor/actions.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    class Processor
+      # Actions store the actions information in memory
+      # Also, takes care of merging when RC send new information
+      module Actions
+        class << self
+          def actions
+            @actions ||= []
+          end
+
+          def fecth_configuration(action)
+            actions.find { |action_configuration| action_configuration['id'] == action }
+          end
+
+          def merge(actions_to_merge)
+            return if actions_to_merge.empty?
+
+            if actions.empty?
+              @actions = actions_to_merge
+            else
+              merged_actions = []
+              actions_dup = actions.dup
+
+              actions_to_merge.each do |new_action|
+                existing_action = actions_dup.find { |action| new_action['id'] == action['id'] }
+
+                # the old action is discard and the new kept
+                actions_dup.delete(existing_action) if existing_action
+                merged_actions << new_action
+              end
+
+              @actions = merged_actions.concat(actions_dup)
+            end
+          end
+
+          private
+
+          # Used in tests
+          def reset
+            @actions = []
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/processor/rule_merger.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../assets'
+
 module Datadog
   module AppSec
     class Processor
@@ -16,8 +18,25 @@ module Datadog
           end
         end
 
+        DEFAULT_WAF_PROCESSORS = begin
+          JSON.parse(Datadog::AppSec::Assets.waf_processors)
+        rescue StandardError => e
+          Datadog.logger.error { "libddwaf rulemerger failed to parse default waf processors. Error: #{e.inspect}" }
+          []
+        end
+
+        DEFAULT_WAF_SCANNERS = begin
+          JSON.parse(Datadog::AppSec::Assets.waf_scanners)
+        rescue StandardError => e
+          Datadog.logger.error { "libddwaf rulemerger failed to parse default waf scanners. Error: #{e.inspect}" }
+          []
+        end
+
         class << self
-          def merge(rules:, data: [], overrides: [], exclusions: [], custom_rules: [])
+          def merge(
+            rules:, data: [], overrides: [], exclusions: [], custom_rules: [],
+            processors: DEFAULT_WAF_PROCESSORS, scanners: DEFAULT_WAF_SCANNERS
+          )
             combined_rules = combine_rules(rules)
 
             combined_data = combine_data(data) if data.any?
@@ -29,7 +48,8 @@ module Datadog
             combined_rules['rules_override'] = combined_overrides if combined_overrides
             combined_rules['exclusions'] = combined_exclusions if combined_exclusions
             combined_rules['custom_rules'] = combined_custom_rules if combined_custom_rules
-
+            combined_rules['processors'] = processors
+            combined_rules['scanners'] = scanners
             combined_rules
           end
 

data/lib/datadog/appsec/processor.rb

@@ -22,8 +22,15 @@ module Datadog
 
           start_ns = Core::Utils::Time.get_time(:nanosecond)
 
-          # this WAF::Context#run call is not thread safe as it mutates the context
-          # TODO: remove multiple assignment
+          input.reject! do |_, v|
+            case v
+            when TrueClass, FalseClass
+              false
+            else
+              v.nil? ? true : v.empty?
+            end
+          end
+
           _code, res = @context.run(input, timeout)
 
           stop_ns = Core::Utils::Time.get_time(:nanosecond)
@@ -38,15 +45,36 @@ module Datadog
           @run_mutex.unlock
         end
 
+        def extract_schema
+          return unless extract_schema?
+
+          input = {
+            'waf.context.processor' => {
+              'extract-schema' => true
+            }
+          }
+
+          _code, res = @context.run(input, WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
+
+          res
+        end
+
         def finalize
           @context.finalize
         end
+
+        private
+
+        def extract_schema?
+          Datadog.configuration.appsec.api_security.enabled &&
+            Datadog.configuration.appsec.api_security.sample_rate.sample?
+        end
       end
 
-      attr_reader :ruleset_info, :addresses
+      attr_reader :diagnostics, :addresses
 
       def initialize(ruleset:)
-        @ruleset_info = nil
+        @diagnostics = nil
         @addresses = []
         settings = Datadog.configuration.appsec
 
@@ -83,7 +111,7 @@ module Datadog
         }
 
         @handle = Datadog::AppSec::WAF::Handle.new(ruleset, obfuscator: obfuscator_config)
-        @ruleset_info = @handle.ruleset_info
+        @diagnostics = @handle.diagnostics
         @addresses = @handle.required_addresses
 
         true
@@ -92,7 +120,7 @@ module Datadog
           "libddwaf failed to initialize, error: #{e.inspect}"
         end
 
-        @ruleset_info = e.ruleset_info if e.ruleset_info
+        @diagnostics = e.diagnostics if e.diagnostics
 
         false
       rescue StandardError => e

data/lib/datadog/appsec/remote.rb

@@ -31,6 +31,7 @@ module Datadog
           CAP_ASM_RESPONSE_BLOCKING,
           CAP_ASM_DD_RULES,
           CAP_ASM_CUSTOM_RULES,
+          CAP_ASM_CUSTOM_BLOCKING_RESPONSE,
         ].freeze
 
         ASM_PRODUCTS = [
@@ -63,6 +64,7 @@ module Datadog
             data = []
             overrides = []
             exclusions = []
+            actions = []
 
             repository.contents.each do |content|
               parsed_content = parse_content(content)
@@ -76,6 +78,7 @@ module Datadog
                 overrides << parsed_content['rules_override'] if parsed_content['rules_override']
                 exclusions << parsed_content['exclusions'] if parsed_content['exclusions']
                 custom_rules << parsed_content['custom_rules'] if parsed_content['custom_rules']
+                actions.concat(parsed_content['actions']) if parsed_content['actions']
               end
             end
 
@@ -95,7 +98,7 @@ module Datadog
               custom_rules: custom_rules,
             )
 
-            Datadog::AppSec.reconfigure(ruleset: ruleset)
+            Datadog::AppSec.reconfigure(ruleset: ruleset, actions: actions)
           end
 
           [receiver]

data/lib/datadog/appsec/response.rb

@@ -28,19 +28,80 @@ module Datadog
       end
 
       class << self
-        def negotiate(env)
+        def negotiate(env, actions)
+          # @type var configured_response: Response?
+          configured_response = nil
+          actions.each do |action|
+            # Need to use next to make steep happy :(
+            # I rather use break to stop the execution
+            next if configured_response
+
+            action_configuration = AppSec::Processor::Actions.fecth_configuration(action)
+            next unless action_configuration
+
+            configured_response = case action_configuration['type']
+                                  when 'block_request'
+                                    block_response(env, action_configuration['parameters'])
+                                  when 'redirect_request'
+                                    redirect_response(env, action_configuration['parameters'])
+                                  end
+          end
+
+          configured_response || default_response(env)
+        end
+
+        private
+
+        def default_response(env)
           content_type = content_type(env)
 
-          Datadog.logger.debug { "negotiated response content type: #{content_type}" }
+          body = []
+          body << content(content_type)
 
           Response.new(
             status: 403,
             headers: { 'Content-Type' => content_type },
-            body: [Datadog::AppSec::Assets.blocked(format: CONTENT_TYPE_TO_FORMAT[content_type])]
+            body: body,
           )
         end
 
-        private
+        def block_response(env, options)
+          content_type = if options['type'] == 'auto'
+                           content_type(env)
+                         else
+                           FORMAT_TO_CONTENT_TYPE[options['type']]
+                         end
+
+          body = []
+          body << content(content_type)
+
+          Response.new(
+            status: options['status_code'] || 403,
+            headers: { 'Content-Type' => content_type },
+            body: body,
+          )
+        end
+
+        def redirect_response(env, options)
+          if options['location'] && !options['location'].empty?
+            content_type = content_type(env)
+
+            status = options['status_code'] >= 300 && options['status_code'] < 400 ? options['status_code'] : 303
+
+            headers = {
+              'Content-Type' => content_type,
+              'Location' => options['location']
+            }
+
+            Response.new(
+              status: status,
+              headers: headers,
+              body: [],
+            )
+          else
+            default_response(env)
+          end
+        end
 
         CONTENT_TYPE_TO_FORMAT = {
           'application/json' => :json,
@@ -48,6 +109,11 @@ module Datadog
           'text/plain' => :text,
         }.freeze
 
+        FORMAT_TO_CONTENT_TYPE = {
+          'json' => 'application/json',
+          'html' => 'text/html',
+        }.freeze
+
         DEFAULT_CONTENT_TYPE = 'application/json'
 
         def content_type(env)
@@ -67,6 +133,18 @@ module Datadog
         rescue Datadog::AppSec::Utils::HTTP::MediaRange::ParseError
           DEFAULT_CONTENT_TYPE
         end
+
+        def content(content_type)
+          content_format = CONTENT_TYPE_TO_FORMAT[content_type]
+
+          using_default = Datadog.configuration.appsec.block.templates.using_default?(content_format)
+
+          if using_default
+            Datadog::AppSec::Assets.blocked(format: content_format)
+          else
+            Datadog.configuration.appsec.block.templates.send(content_format)
+          end
+        end
       end
     end
   end

data/lib/datadog/appsec/sample_rate.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    # SampleRate basic sample rate
+    class SampleRate
+      attr_reader :rate
+
+      def initialize(rate)
+        @rate = rate
+      end
+
+      def sample?
+        return false if rate <= 0
+        return true if rate >= 1
+
+        Kernel.rand < rate
+      end
+    end
+  end
+end

data/lib/datadog/appsec.rb

@@ -23,12 +23,12 @@ module Datadog
         appsec_component.processor if appsec_component
       end
 
-      def reconfigure(ruleset:)
+      def reconfigure(ruleset:, actions:)
         appsec_component = components.appsec
 
         return unless appsec_component
 
-        appsec_component.reconfigure(ruleset: ruleset)
+        appsec_component.reconfigure(ruleset: ruleset, actions: actions)
       end
 
       def reconfigure_lock(&block)

data/lib/datadog/core/configuration/agent_settings_resolver.rb

@@ -1,8 +1,8 @@
 require 'uri'
 
 require_relative 'settings'
-require_relative '../../tracing/configuration/ext'
-require_relative '../../../ddtrace/transport/ext'
+require_relative 'ext'
+require_relative '../transport/ext'
 
 module Datadog
   module Core
@@ -16,6 +16,7 @@ module Datadog
       #
       # Whenever there is a conflict (different configurations are provided in different orders), it MUST warn the users
       # about it and pick a value based on the following priority: code > environment variable > defaults.
+      # DEV-2.0: The deprecated_for_removal_transport_configuration_proc should be removed.
       class AgentSettingsResolver
         AgentSettings = \
           Struct.new(
@@ -66,9 +67,9 @@ module Datadog
 
         def call
           # A transport_options proc configured for unix domain socket overrides most of the logic on this file
-          if transport_options.adapter == Datadog::Transport::Ext::UnixSocket::ADAPTER
+          if transport_options.adapter == Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
             return AgentSettings.new(
-              adapter: Datadog::Transport::Ext::UnixSocket::ADAPTER,
+              adapter: Datadog::Core::Transport::Ext::UnixSocket::ADAPTER,
               ssl: false,
               hostname: nil,
               port: nil,
@@ -96,9 +97,9 @@ module Datadog
 
         def adapter
           if should_use_uds?
-            Datadog::Transport::Ext::UnixSocket::ADAPTER
+            Datadog::Core::Transport::Ext::UnixSocket::ADAPTER
           else
-            Datadog::Transport::Ext::HTTP::ADAPTER
+            Datadog::Core::Transport::Ext::HTTP::ADAPTER
           end
         end
 
@@ -115,7 +116,7 @@ module Datadog
               value: settings.agent.host
             ),
             DetectedConfiguration.new(
-              friendly_name: "#{Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_URL} environment variable",
+              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_URL} environment variable",
               value: parsed_http_url && parsed_http_url.hostname
             ),
             DetectedConfiguration.new(
@@ -138,12 +139,12 @@ module Datadog
               value: settings.agent.port,
             ),
             DetectedConfiguration.new(
-              friendly_name: "#{Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_URL} environment variable",
+              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_URL} environment variable",
               value: parsed_http_url && parsed_http_url.port,
             ),
             try_parsing_as_integer(
-              friendly_name: "#{Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_PORT} environment variable",
-              value: ENV[Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_PORT],
+              friendly_name: "#{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_PORT} environment variable",
+              value: ENV[Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_PORT],
             )
           )
         end
@@ -167,11 +168,11 @@ module Datadog
         end
 
         def hostname
-          configured_hostname || (should_use_uds? ? nil : Datadog::Transport::Ext::HTTP::DEFAULT_HOST)
+          configured_hostname || (should_use_uds? ? nil : Datadog::Core::Configuration::Ext::Agent::HTTP::DEFAULT_HOST)
         end
 
         def port
-          configured_port || (should_use_uds? ? nil : Datadog::Transport::Ext::HTTP::DEFAULT_PORT)
+          configured_port || (should_use_uds? ? nil : Datadog::Core::Configuration::Ext::Agent::HTTP::DEFAULT_PORT)
         end
 
         # Unix socket path in the file system
@@ -201,8 +202,12 @@ module Datadog
         # In transport_options, we try to invoke the transport_options proc and get its configuration. In case that
         # doesn't work, we include the proc directly in the agent settings result.
         def deprecated_for_removal_transport_configuration_proc
-          if settings.tracing.transport_options.is_a?(Proc) && transport_options.adapter.nil?
-            settings.tracing.transport_options
+          transport_options_settings if transport_options_settings.is_a?(Proc) && transport_options.adapter.nil?
+        end
+
+        def transport_options_settings
+          @transport_options_settings ||= begin
+            settings.tracing.transport_options if settings.respond_to?(:tracing) && settings.tracing
           end
         end
 
@@ -215,9 +220,9 @@ module Datadog
             if configured_hostname.nil? &&
                 configured_port.nil? &&
                 deprecated_for_removal_transport_configuration_proc.nil? &&
-                File.exist?(Datadog::Transport::Ext::UnixSocket::DEFAULT_PATH)
+                File.exist?(Datadog::Core::Configuration::Ext::Agent::UnixSocket::DEFAULT_PATH)
 
-              Datadog::Transport::Ext::UnixSocket::DEFAULT_PATH
+              Datadog::Core::Configuration::Ext::Agent::UnixSocket::DEFAULT_PATH
             end
         end
 
@@ -235,7 +240,7 @@ module Datadog
         def parsed_url
           return @parsed_url if defined?(@parsed_url)
 
-          unparsed_url_from_env = ENV[Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_URL]
+          unparsed_url_from_env = ENV[Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_URL]
 
           @parsed_url =
             if unparsed_url_from_env
@@ -246,9 +251,9 @@ module Datadog
               else
                 # rubocop:disable Layout/LineLength
                 log_warning(
-                  "Invalid URI scheme '#{parsed.scheme}' for #{Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_URL} " \
+                  "Invalid URI scheme '#{parsed.scheme}' for #{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_URL} " \
                   "environment variable ('#{unparsed_url_from_env}'). " \
-                  "Ignoring the contents of #{Datadog::Tracing::Configuration::Ext::Transport::ENV_DEFAULT_URL}."
+                  "Ignoring the contents of #{Datadog::Core::Configuration::Ext::Agent::ENV_DEFAULT_URL}."
                 )
                 # rubocop:enable Layout/LineLength
 
@@ -328,7 +333,7 @@ module Datadog
         def transport_options
           return @transport_options if defined?(@transport_options)
 
-          transport_options_proc = settings.tracing.transport_options
+          transport_options_proc = transport_options_settings
 
           @transport_options = TransportOptions.new
 
@@ -380,14 +385,14 @@ module Datadog
 
           def adapter(kind_or_custom_adapter, *args, **kwargs)
             case kind_or_custom_adapter
-            when Datadog::Transport::Ext::HTTP::ADAPTER
-              @transport_options.adapter = Datadog::Transport::Ext::HTTP::ADAPTER
+            when Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
+              @transport_options.adapter = Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
               @transport_options.hostname = args[0] || kwargs[:hostname]
               @transport_options.port = args[1] || kwargs[:port]
               @transport_options.timeout_seconds = kwargs[:timeout]
               @transport_options.ssl = kwargs[:ssl]
-            when Datadog::Transport::Ext::UnixSocket::ADAPTER
-              @transport_options.adapter = Datadog::Transport::Ext::UnixSocket::ADAPTER
+            when Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
+              @transport_options.adapter = Datadog::Core::Configuration::Ext::Agent::UnixSocket::ADAPTER
               @transport_options.uds_path = args[0] || kwargs[:uds_path]
             end
 

data/lib/datadog/core/configuration/base.rb

@@ -1,4 +1,3 @@
-require_relative '../environment/variable_helpers'
 require_relative 'options'
 
 module Datadog
@@ -8,8 +7,6 @@ module Datadog
       # @public_api
       module Base
         def self.included(base)
-          base.extend(Core::Environment::VariableHelpers)
-          base.include(Core::Environment::VariableHelpers)
           base.include(Options)
 
           base.extend(ClassMethods)
@@ -57,14 +54,7 @@ module Datadog
           end
 
           def configure(opts = {})
-            # Sort the options in preference of dependency order first
-            ordering = self.class.options.dependency_order
-            sorted_opts = opts.sort_by do |name, _value|
-              ordering.index(name) || (ordering.length + 1)
-            end.to_h
-
-            # Apply options in sort order
-            sorted_opts.each do |name, value|
+            opts.each do |name, value|
               if respond_to?("#{name}=")
                 send("#{name}=", value)
               elsif option_defined?(name)

data/lib/datadog/core/configuration/components.rb

@@ -1,4 +1,5 @@
 require_relative 'agent_settings_resolver'
+require_relative 'ext'
 require_relative '../diagnostics/environment_logger'
 require_relative '../diagnostics/health'
 require_relative '../logger'
@@ -54,7 +55,7 @@ module Datadog
 
           def build_telemetry(settings, agent_settings, logger)
             enabled = settings.telemetry.enabled
-            if agent_settings.adapter != Datadog::Transport::Ext::HTTP::ADAPTER
+            if agent_settings.adapter != Datadog::Core::Configuration::Ext::Agent::HTTP::ADAPTER
               enabled = false
               logger.debug { "Telemetry disabled. Agent network adapter not supported: #{agent_settings.adapter}" }
             end
@@ -81,10 +82,14 @@ module Datadog
         def initialize(settings)
           @logger = self.class.build_logger(settings)
 
+          # This agent_settings is intended for use within Core. If you require
+          # agent_settings within a product outside of core you should extend
+          # the Core resolver from within your product/component's namespace.
           agent_settings = AgentSettingsResolver.call(settings, logger: @logger)
 
           @remote = Remote::Component.build(settings, agent_settings)
-          @tracer = self.class.build_tracer(settings, agent_settings)
+          @tracer = self.class.build_tracer(settings, logger: @logger)
+
           @profiler = Datadog::Profiling::Component.build_profiler_component(
             settings: settings,
             agent_settings: agent_settings,

data/lib/datadog/core/configuration/ext.rb

@@ -17,9 +17,30 @@ module Datadog
           ENV_DEFAULT_PORT = 'DD_METRIC_AGENT_PORT'
         end
 
+        # DEV-2.0: This module only exists for backwards compatibility with the public API.
+        # It should be consolidated into the Agent module below.
         module Transport
           ENV_DEFAULT_HOST = 'DD_AGENT_HOST'
         end
+
+        module Agent
+          # env var has "trace" in it, but it really applies to all products
+          ENV_DEFAULT_PORT = 'DD_TRACE_AGENT_PORT'
+          ENV_DEFAULT_URL = 'DD_TRACE_AGENT_URL'
+
+          module HTTP
+            ADAPTER = :net_http # DEV: Rename to simply `:http`, as Net::HTTP is an implementation detail.
+            DEFAULT_HOST = '127.0.0.1'
+            DEFAULT_PORT = 8126
+          end
+
+          # @public_api
+          module UnixSocket
+            ADAPTER = :unix
+            DEFAULT_PATH = '/var/run/datadog/apm.socket'
+            DEFAULT_TIMEOUT_SECONDS = 1
+          end
+        end
       end
     end
   end

data/lib/datadog/core/configuration/option.rb

@@ -113,8 +113,6 @@ module Datadog
         def get
           if @is_set
             @value
-          elsif definition.delegate_to
-            context_eval(&definition.delegate_to)
           else
             set_value_from_env_or_default
           end
@@ -141,7 +139,7 @@ module Datadog
           if definition.default.instance_of?(Proc)
             context_eval(&definition.default)
           else
-            definition.experimental_default_proc || Core::Utils::SafeDup.frozen_or_dup(definition.default)
+            definition.default_proc || Core::Utils::SafeDup.frozen_or_dup(definition.default)
           end
         end
 
@@ -266,7 +264,7 @@ module Datadog
             # when restoring a value from `@value_per_precedence`, and we are only running `definition.setter`
             # on the original value, not on a valud that has already been processed by `definition.setter`.
             @value_per_precedence[precedence] = value
-            context_exec(v, old_value, &definition.on_set) if definition.on_set
+            context_exec(v, old_value, &definition.after_set) if definition.after_set
           end
         end