datadog 2.28.0 → 2.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +87 -1
  3. data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +82 -12
  4. data/ext/datadog_profiling_native_extension/collectors_thread_context.c +32 -11
  5. data/ext/datadog_profiling_native_extension/collectors_thread_context.h +3 -1
  6. data/ext/datadog_profiling_native_extension/extconf.rb +9 -24
  7. data/ext/datadog_profiling_native_extension/heap_recorder.c +186 -55
  8. data/ext/datadog_profiling_native_extension/heap_recorder.h +12 -1
  9. data/ext/datadog_profiling_native_extension/http_transport.c +51 -64
  10. data/ext/datadog_profiling_native_extension/native_extension_helpers.rb +0 -13
  11. data/ext/datadog_profiling_native_extension/profiling.c +3 -1
  12. data/ext/datadog_profiling_native_extension/setup_signal_handler.c +24 -8
  13. data/ext/datadog_profiling_native_extension/setup_signal_handler.h +1 -3
  14. data/ext/datadog_profiling_native_extension/stack_recorder.c +63 -48
  15. data/ext/datadog_profiling_native_extension/stack_recorder.h +2 -1
  16. data/ext/libdatadog_api/crashtracker.c +5 -0
  17. data/ext/libdatadog_api/crashtracker_report_exception.c +126 -0
  18. data/ext/libdatadog_api/extconf.rb +4 -21
  19. data/ext/libdatadog_extconf_helpers.rb +49 -11
  20. data/lib/datadog/ai_guard/configuration/settings.rb +3 -0
  21. data/lib/datadog/appsec/assets/blocked.html +2 -1
  22. data/lib/datadog/appsec/configuration/settings.rb +14 -0
  23. data/lib/datadog/appsec/context.rb +44 -9
  24. data/lib/datadog/appsec/contrib/active_record/patcher.rb +3 -0
  25. data/lib/datadog/appsec/contrib/devise/integration.rb +1 -1
  26. data/lib/datadog/appsec/contrib/excon/patcher.rb +2 -0
  27. data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb +55 -6
  28. data/lib/datadog/appsec/contrib/faraday/integration.rb +1 -1
  29. data/lib/datadog/appsec/contrib/faraday/patcher.rb +1 -1
  30. data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb +60 -7
  31. data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb +11 -6
  32. data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb +1 -1
  33. data/lib/datadog/appsec/contrib/graphql/integration.rb +1 -1
  34. data/lib/datadog/appsec/contrib/rack/gateway/request.rb +6 -10
  35. data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +4 -4
  36. data/lib/datadog/appsec/contrib/rack/integration.rb +1 -1
  37. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +26 -5
  38. data/lib/datadog/appsec/contrib/rack/response_body.rb +36 -0
  39. data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +2 -2
  40. data/lib/datadog/appsec/contrib/rails/integration.rb +1 -1
  41. data/lib/datadog/appsec/contrib/rails/patches/process_action_patch.rb +2 -0
  42. data/lib/datadog/appsec/contrib/rest_client/patcher.rb +2 -0
  43. data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb +72 -7
  44. data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +5 -3
  45. data/lib/datadog/appsec/counter_sampler.rb +25 -0
  46. data/lib/datadog/appsec/event.rb +1 -17
  47. data/lib/datadog/appsec/instrumentation/gateway/middleware.rb +2 -3
  48. data/lib/datadog/appsec/instrumentation/gateway.rb +2 -2
  49. data/lib/datadog/appsec/metrics/telemetry_exporter.rb +18 -0
  50. data/lib/datadog/appsec/monitor/gateway/watcher.rb +2 -2
  51. data/lib/datadog/appsec/security_engine/engine.rb +23 -2
  52. data/lib/datadog/appsec/utils/http/body.rb +38 -0
  53. data/lib/datadog/appsec/utils/http/media_range.rb +2 -1
  54. data/lib/datadog/appsec/utils/http/media_type.rb +28 -35
  55. data/lib/datadog/appsec/utils/http/url_encoded.rb +52 -0
  56. data/lib/datadog/core/configuration/components.rb +29 -4
  57. data/lib/datadog/core/configuration/option.rb +2 -1
  58. data/lib/datadog/core/configuration/options.rb +1 -1
  59. data/lib/datadog/core/configuration/settings.rb +27 -3
  60. data/lib/datadog/core/configuration/supported_configurations.rb +19 -0
  61. data/lib/datadog/core/configuration.rb +2 -2
  62. data/lib/datadog/core/crashtracking/component.rb +71 -19
  63. data/lib/datadog/core/environment/agent_info.rb +65 -1
  64. data/lib/datadog/core/logger.rb +1 -1
  65. data/lib/datadog/core/metrics/logging.rb +1 -1
  66. data/lib/datadog/core/process_discovery.rb +20 -19
  67. data/lib/datadog/core/rate_limiter.rb +2 -0
  68. data/lib/datadog/core/remote/component.rb +16 -5
  69. data/lib/datadog/core/remote/transport/config.rb +5 -11
  70. data/lib/datadog/core/runtime/metrics.rb +1 -2
  71. data/lib/datadog/core/telemetry/component.rb +0 -13
  72. data/lib/datadog/core/telemetry/transport/telemetry.rb +5 -6
  73. data/lib/datadog/core/transport/ext.rb +1 -0
  74. data/lib/datadog/core/transport/http/response.rb +4 -0
  75. data/lib/datadog/core/transport/parcel.rb +61 -9
  76. data/lib/datadog/core/utils/base64.rb +1 -1
  77. data/lib/datadog/core/utils/fnv.rb +26 -0
  78. data/lib/datadog/core/workers/interval_loop.rb +13 -6
  79. data/lib/datadog/core/workers/queue.rb +0 -4
  80. data/lib/datadog/core/workers/runtime_metrics.rb +9 -1
  81. data/lib/datadog/core.rb +6 -1
  82. data/lib/datadog/data_streams/processor.rb +35 -33
  83. data/lib/datadog/data_streams/transport/http/stats.rb +6 -0
  84. data/lib/datadog/data_streams/transport/http.rb +0 -4
  85. data/lib/datadog/data_streams/transport/stats.rb +5 -12
  86. data/lib/datadog/di/boot.rb +1 -0
  87. data/lib/datadog/di/component.rb +17 -5
  88. data/lib/datadog/di/configuration/settings.rb +9 -0
  89. data/lib/datadog/di/context.rb +6 -0
  90. data/lib/datadog/di/instrumenter.rb +183 -134
  91. data/lib/datadog/di/probe.rb +10 -1
  92. data/lib/datadog/di/probe_file_loader.rb +2 -2
  93. data/lib/datadog/di/probe_manager.rb +86 -64
  94. data/lib/datadog/di/probe_notification_builder.rb +46 -18
  95. data/lib/datadog/di/probe_notifier_worker.rb +65 -9
  96. data/lib/datadog/di/probe_repository.rb +198 -0
  97. data/lib/datadog/di/proc_responder.rb +4 -0
  98. data/lib/datadog/di/remote.rb +6 -7
  99. data/lib/datadog/di/serializer.rb +127 -9
  100. data/lib/datadog/di/transport/diagnostics.rb +5 -7
  101. data/lib/datadog/di/transport/http/diagnostics.rb +3 -1
  102. data/lib/datadog/di/transport/http/input.rb +1 -1
  103. data/lib/datadog/di/transport/http.rb +12 -3
  104. data/lib/datadog/di/transport/input.rb +51 -14
  105. data/lib/datadog/kit/tracing/method_tracer.rb +132 -0
  106. data/lib/datadog/open_feature/configuration.rb +2 -0
  107. data/lib/datadog/open_feature/transport.rb +8 -11
  108. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +13 -0
  109. data/lib/datadog/profiling/component.rb +20 -6
  110. data/lib/datadog/profiling/http_transport.rb +5 -6
  111. data/lib/datadog/profiling/profiler.rb +15 -8
  112. data/lib/datadog/tracing/contrib/dalli/integration.rb +4 -1
  113. data/lib/datadog/tracing/contrib/ethon/configuration/settings.rb +5 -1
  114. data/lib/datadog/tracing/contrib/ethon/ext.rb +1 -0
  115. data/lib/datadog/tracing/contrib/excon/configuration/settings.rb +5 -2
  116. data/lib/datadog/tracing/contrib/excon/ext.rb +1 -0
  117. data/lib/datadog/tracing/contrib/faraday/configuration/settings.rb +5 -2
  118. data/lib/datadog/tracing/contrib/faraday/ext.rb +1 -0
  119. data/lib/datadog/tracing/contrib/grape/endpoint.rb +2 -2
  120. data/lib/datadog/tracing/contrib/grape/instrumentation.rb +13 -8
  121. data/lib/datadog/tracing/contrib/grape/patcher.rb +6 -1
  122. data/lib/datadog/tracing/contrib/grpc/configuration/settings.rb +5 -2
  123. data/lib/datadog/tracing/contrib/grpc/ext.rb +1 -0
  124. data/lib/datadog/tracing/contrib/http/configuration/settings.rb +5 -2
  125. data/lib/datadog/tracing/contrib/http/ext.rb +1 -0
  126. data/lib/datadog/tracing/contrib/http/integration.rb +0 -2
  127. data/lib/datadog/tracing/contrib/httpclient/configuration/settings.rb +5 -2
  128. data/lib/datadog/tracing/contrib/httpclient/ext.rb +1 -0
  129. data/lib/datadog/tracing/contrib/httprb/configuration/settings.rb +5 -2
  130. data/lib/datadog/tracing/contrib/httprb/ext.rb +1 -0
  131. data/lib/datadog/tracing/contrib/karafka/configuration/settings.rb +5 -1
  132. data/lib/datadog/tracing/contrib/karafka/ext.rb +1 -0
  133. data/lib/datadog/tracing/contrib/mysql2/configuration/settings.rb +6 -0
  134. data/lib/datadog/tracing/contrib/mysql2/instrumentation.rb +2 -1
  135. data/lib/datadog/tracing/contrib/pg/configuration/settings.rb +6 -0
  136. data/lib/datadog/tracing/contrib/pg/instrumentation.rb +2 -1
  137. data/lib/datadog/tracing/contrib/propagation/sql_comment/ext.rb +10 -0
  138. data/lib/datadog/tracing/contrib/propagation/sql_comment/mode.rb +5 -1
  139. data/lib/datadog/tracing/contrib/propagation/sql_comment.rb +24 -0
  140. data/lib/datadog/tracing/contrib/que/configuration/settings.rb +5 -2
  141. data/lib/datadog/tracing/contrib/que/ext.rb +1 -0
  142. data/lib/datadog/tracing/contrib/rack/configuration/settings.rb +5 -1
  143. data/lib/datadog/tracing/contrib/rack/ext.rb +1 -0
  144. data/lib/datadog/tracing/contrib/rack/route_inference.rb +18 -6
  145. data/lib/datadog/tracing/contrib/rails/configuration/settings.rb +5 -2
  146. data/lib/datadog/tracing/contrib/rails/ext.rb +1 -0
  147. data/lib/datadog/tracing/contrib/registerable.rb +11 -0
  148. data/lib/datadog/tracing/contrib/rest_client/configuration/settings.rb +5 -2
  149. data/lib/datadog/tracing/contrib/rest_client/ext.rb +1 -0
  150. data/lib/datadog/tracing/contrib/sidekiq/configuration/settings.rb +5 -1
  151. data/lib/datadog/tracing/contrib/sidekiq/ext.rb +1 -0
  152. data/lib/datadog/tracing/contrib/sinatra/configuration/settings.rb +5 -1
  153. data/lib/datadog/tracing/contrib/sinatra/ext.rb +1 -0
  154. data/lib/datadog/tracing/contrib/sneakers/integration.rb +15 -4
  155. data/lib/datadog/tracing/contrib/trilogy/configuration/settings.rb +6 -0
  156. data/lib/datadog/tracing/contrib/trilogy/instrumentation.rb +3 -1
  157. data/lib/datadog/tracing/contrib/waterdrop/configuration/settings.rb +5 -1
  158. data/lib/datadog/tracing/contrib/waterdrop/ext.rb +1 -0
  159. data/lib/datadog/tracing/metadata/ext.rb +4 -0
  160. data/lib/datadog/tracing/sync_writer.rb +0 -1
  161. data/lib/datadog/tracing/transport/io/client.rb +5 -8
  162. data/lib/datadog/tracing/transport/io/traces.rb +28 -34
  163. data/lib/datadog/tracing/transport/trace_formatter.rb +11 -0
  164. data/lib/datadog/tracing/transport/traces.rb +4 -10
  165. data/lib/datadog/tracing/writer.rb +0 -1
  166. data/lib/datadog/version.rb +1 -1
  167. metadata +14 -8
  168. data/lib/datadog/appsec/contrib/rails/ext.rb +0 -13
  169. data/lib/datadog/tracing/workers/trace_writer.rb +0 -204
@@ -100,7 +100,8 @@
100
100
  <footer>
101
101
  <p>Security provided by <a
102
102
  href="https://www.datadoghq.com/product/security-platform/application-security-monitoring/"
103
- target="_blank">Datadog</a></p>
103
+ target="_blank"
104
+ rel="noopener noreferrer">Datadog</a></p>
104
105
  </footer>
105
106
  </body>
106
107
 
@@ -393,6 +393,20 @@ module Datadog
393
393
  end
394
394
  end
395
395
  end
396
+
397
+ settings :downstream_body_analysis do
398
+ option :sample_rate do |o|
399
+ o.type :float
400
+ o.env 'DD_API_SECURITY_DOWNSTREAM_BODY_ANALYSIS_SAMPLE_RATE'
401
+ o.default 0.5
402
+ end
403
+
404
+ option :max_requests do |o|
405
+ o.type :int
406
+ o.env 'DD_API_SECURITY_MAX_DOWNSTREAM_REQUEST_BODY_ANALYSIS'
407
+ o.default 1
408
+ end
409
+ end
396
410
  end
397
411
 
398
412
  option :sca_enabled do |o|
@@ -1,17 +1,36 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative 'counter_sampler'
3
4
  require_relative 'metrics'
5
+ require_relative 'security_event'
4
6
 
5
7
  module Datadog
6
8
  module AppSec
7
- # This class accumulates the context over the request life-cycle and exposes
8
- # interface sufficient for instrumentation to perform threat detection.
9
+ # Request-bound context providing threat detection interface.
10
+ #
11
+ # Activated at the start of a request (see `Contrib::Rack::RequestMiddleware`)
12
+ # and shared across all instrumentations within that request's lifecycle.
13
+ #
14
+ # Accumulates security events, metrics, and state needed for coordinated
15
+ # threat detection.
16
+ #
17
+ # @api private
9
18
  class Context
10
19
  # Steep: https://github.com/soutaro/steep/issues/1880
11
20
  ActiveContextError = Class.new(StandardError) # steep:ignore IncompatibleAssignment
12
21
 
13
22
  # TODO: add delegators for active trace span
14
- attr_reader :trace, :span, :events
23
+ attr_reader :trace, :span
24
+
25
+ # Shared mutable storage for counters, flags, and data accumulated during
26
+ # the request's lifecycle.
27
+ #
28
+ # NOTE: This attribute is a subject to change, but in a current form
29
+ # it's a `Hash`-like structure.
30
+ attr_reader :state
31
+
32
+ # Sampler for downstream HTTP request/response body analysis.
33
+ attr_reader :downstream_body_sampler
15
34
 
16
35
  class << self
17
36
  def activate(context)
@@ -35,10 +54,16 @@ module Datadog
35
54
  def initialize(trace, span, waf_runner)
36
55
  @trace = trace
37
56
  @span = span
38
- @events = []
39
57
  @waf_runner = waf_runner
40
58
  @metrics = Metrics::Collector.new
41
- @interrupted = false
59
+ @downstream_body_sampler = CounterSampler.new(
60
+ Datadog.configuration.appsec.api_security.downstream_body_analysis.sample_rate
61
+ )
62
+ @state = {
63
+ events: [],
64
+ interrupted: false,
65
+ downstream_body_analyzed_count: 0
66
+ }
42
67
  end
43
68
 
44
69
  def run_waf(persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
@@ -57,12 +82,16 @@ module Datadog
57
82
  result
58
83
  end
59
84
 
85
+ def events
86
+ @state[:events]
87
+ end
88
+
60
89
  def mark_as_interrupted!
61
- @interrupted = true
90
+ @state[:interrupted] = true
62
91
  end
63
92
 
64
93
  def interrupted?
65
- @interrupted
94
+ @state[:interrupted]
66
95
  end
67
96
 
68
97
  def waf_runner_ruleset_version
@@ -73,8 +102,13 @@ module Datadog
73
102
  @waf_runner.waf_addresses
74
103
  end
75
104
 
76
- def extract_schema
77
- @waf_runner.run({'waf.context.processor' => {'extract-schema' => true}}, {})
105
+ def extract_schema!
106
+ waf_result = @waf_runner.run({'waf.context.processor' => {'extract-schema' => true}}, {})
107
+ security_event = AppSec::SecurityEvent.new(waf_result, trace: trace, span: span)
108
+
109
+ @state[:schema_extracted] = security_event.schema?
110
+
111
+ events.push(security_event)
78
112
  end
79
113
 
80
114
  def export_metrics
@@ -88,6 +122,7 @@ module Datadog
88
122
  return if @trace.nil?
89
123
 
90
124
  Metrics::TelemetryExporter.export_waf_request_metrics(@metrics.waf, self)
125
+ Metrics::TelemetryExporter.export_api_security_metrics(self)
91
126
  end
92
127
 
93
128
  def finalize!
@@ -61,6 +61,7 @@ module Datadog
61
61
  end
62
62
 
63
63
  ::ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(instrumentation_module)
64
+ Patcher.instance_variable_set(:@patched, true)
64
65
  end
65
66
 
66
67
  def patch_mysql2_adapter
@@ -73,6 +74,7 @@ module Datadog
73
74
  end
74
75
 
75
76
  ::ActiveRecord::ConnectionAdapters::Mysql2Adapter.prepend(instrumentation_module)
77
+ Patcher.instance_variable_set(:@patched, true)
76
78
  end
77
79
 
78
80
  def patch_postgresql_adapter
@@ -93,6 +95,7 @@ module Datadog
93
95
  end
94
96
 
95
97
  ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(instrumentation_module)
98
+ Patcher.instance_variable_set(:@patched, true)
96
99
  end
97
100
  end
98
101
  end
@@ -24,7 +24,7 @@ module Datadog
24
24
  end
25
25
 
26
26
  def self.compatible?
27
- super && version >= MINIMUM_VERSION
27
+ !!(super && (version&.>= MINIMUM_VERSION))
28
28
  end
29
29
 
30
30
  def self.auto_instrument?
@@ -20,6 +20,8 @@ module Datadog
20
20
  require_relative 'ssrf_detection_middleware'
21
21
 
22
22
  ::Excon.defaults[:middlewares].insert(0, SSRFDetectionMiddleware)
23
+
24
+ Patcher.instance_variable_set(:@patched, true)
23
25
  end
24
26
  end
25
27
  end
@@ -5,6 +5,8 @@ require 'excon'
5
5
  require_relative '../../event'
6
6
  require_relative '../../trace_keeper'
7
7
  require_relative '../../security_event'
8
+ require_relative '../../utils/http/url_encoded'
9
+ require_relative '../../utils/http/body'
8
10
 
9
11
  module Datadog
10
12
  module AppSec
@@ -12,17 +14,36 @@ module Datadog
12
14
  module Excon
13
15
  # AppSec Middleware for Excon
14
16
  class SSRFDetectionMiddleware < ::Excon::Middleware::Base
17
+ REDIRECT_STATUS_CODES = (300..399).freeze
18
+ SAMPLE_BODY_KEY = :__datadog_appsec_sample_downstream_body
19
+
15
20
  def request_call(data)
16
21
  context = AppSec.active_context
17
22
  return super unless context && AppSec.rasp_enabled?
18
23
 
19
- timeout = Datadog.configuration.appsec.waf_timeout
24
+ url = request_url(data)
25
+ headers = normalize_headers(data[:headers])
26
+ # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
20
27
  ephemeral_data = {
21
- 'server.io.net.url' => request_url(data),
28
+ 'server.io.net.url' => url,
22
29
  'server.io.net.request.method' => data[:method].to_s.upcase,
23
- 'server.io.net.request.headers' => normalize_headers(data[:headers])
30
+ 'server.io.net.request.headers' => headers
24
31
  }
25
32
 
33
+ is_redirect = context.state[:downstream_redirect_url] == url
34
+ if is_redirect
35
+ context.state.delete(:downstream_redirect_url)
36
+ data[SAMPLE_BODY_KEY] = true
37
+ else
38
+ mark_body_sampling!(data, context: context)
39
+ end
40
+
41
+ if !is_redirect && data[SAMPLE_BODY_KEY]
42
+ body = parse_body(data[:body], content_type: headers['content-type'])
43
+ ephemeral_data['server.io.net.request.body'] = body if body
44
+ end
45
+
46
+ timeout = Datadog.configuration.appsec.waf_timeout
26
47
  result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_REQUEST_PHASE)
27
48
  handle(result, context: context) if result.match?
28
49
 
@@ -33,12 +54,24 @@ module Datadog
33
54
  context = AppSec.active_context
34
55
  return super unless context && AppSec.rasp_enabled?
35
56
 
36
- timeout = Datadog.configuration.appsec.waf_timeout
57
+ headers = normalize_headers(data.dig(:response, :headers))
58
+ # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
37
59
  ephemeral_data = {
38
60
  'server.io.net.response.status' => data.dig(:response, :status).to_s,
39
- 'server.io.net.response.headers' => normalize_headers(data.dig(:response, :headers))
61
+ 'server.io.net.response.headers' => headers
40
62
  }
41
63
 
64
+ is_redirect = REDIRECT_STATUS_CODES.cover?(data.dig(:response, :status)) && headers.key?('location')
65
+ if is_redirect && data[SAMPLE_BODY_KEY]
66
+ context.state[:downstream_redirect_url] = URI.join(request_url(data), headers['location']).to_s
67
+ end
68
+
69
+ if !is_redirect && data[SAMPLE_BODY_KEY]
70
+ body = parse_body(data.dig(:response, :body), content_type: headers['content-type'])
71
+ ephemeral_data['server.io.net.response.body'] = body if body
72
+ end
73
+
74
+ timeout = Datadog.configuration.appsec.waf_timeout
42
75
  result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_RESPONSE_PHASE)
43
76
  handle(result, context: context) if result.match?
44
77
 
@@ -47,9 +80,25 @@ module Datadog
47
80
 
48
81
  private
49
82
 
83
+ def mark_body_sampling!(data, context:)
84
+ max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_requests
85
+ return if context.state[:downstream_body_analyzed_count] >= max
86
+ return unless context.downstream_body_sampler.sample?
87
+
88
+ context.state[:downstream_body_analyzed_count] += 1
89
+ data[SAMPLE_BODY_KEY] = true
90
+ end
91
+
92
+ def parse_body(body, content_type:)
93
+ media_type = Utils::HTTP::MediaType.parse(content_type)
94
+ return unless media_type
95
+
96
+ Utils::HTTP::Body.parse(body, media_type: media_type)
97
+ end
98
+
50
99
  def request_url(data)
51
100
  klass = (data[:scheme] == 'https') ? URI::HTTPS : URI::HTTP
52
- klass.build(host: data[:host], path: data[:path], query: data[:query]).to_s
101
+ klass.build(host: data[:host], port: data[:port], path: data[:path], query: data[:query]).to_s
53
102
  end
54
103
 
55
104
  def normalize_headers(headers)
@@ -25,7 +25,7 @@ module Datadog
25
25
  end
26
26
 
27
27
  def self.compatible?
28
- super && version >= MINIMUM_VERSION
28
+ super && !!(version&.>= MINIMUM_VERSION)
29
29
  end
30
30
 
31
31
  def self.auto_instrument?
@@ -28,7 +28,7 @@ module Datadog
28
28
  end
29
29
 
30
30
  def configure_default_faraday_connection
31
- if target_version >= Gem::Version.new('1.0.0')
31
+ if target_version&.>= Gem::Version.new('1.0.0')
32
32
  # Patch the default connection (e.g. +Faraday.get+)
33
33
  ::Faraday.default_connection.use(:datadog_appsec)
34
34
 
@@ -3,6 +3,8 @@
3
3
  require_relative '../../event'
4
4
  require_relative '../../trace_keeper'
5
5
  require_relative '../../security_event'
6
+ require_relative '../../utils/http/media_type'
7
+ require_relative '../../utils/http/body'
6
8
 
7
9
  module Datadog
8
10
  module AppSec
@@ -10,17 +12,36 @@ module Datadog
10
12
  module Faraday
11
13
  # AppSec SSRF detection Middleware for Faraday
12
14
  class SSRFDetectionMiddleware < ::Faraday::Middleware
15
+ REDIRECT_STATUS_CODES = (300..399).freeze
16
+ SAMPLE_BODY_KEY = :__datadog_appsec_sample_downstream_body
17
+
13
18
  def call(env)
14
19
  context = AppSec.active_context
15
20
  return @app.call(env) unless context && AppSec.rasp_enabled?
16
21
 
17
- timeout = Datadog.configuration.appsec.waf_timeout
22
+ url = env.url.to_s
23
+ headers = normalize_headers(env.request_headers)
24
+ # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
18
25
  ephemeral_data = {
19
- 'server.io.net.url' => env.url.to_s,
26
+ 'server.io.net.url' => url,
20
27
  'server.io.net.request.method' => env.method.to_s.upcase,
21
- 'server.io.net.request.headers' => env.request_headers.transform_keys(&:downcase)
28
+ 'server.io.net.request.headers' => headers
22
29
  }
23
30
 
31
+ is_redirect = context.state[:downstream_redirect_url] == url
32
+ if is_redirect
33
+ context.state.delete(:downstream_redirect_url)
34
+ env[SAMPLE_BODY_KEY] = true
35
+ else
36
+ mark_body_sampling!(env, context: context)
37
+ end
38
+
39
+ if !is_redirect && env[SAMPLE_BODY_KEY]
40
+ body = parse_body(env.body, content_type: headers['content-type'])
41
+ ephemeral_data['server.io.net.request.body'] = body if body
42
+ end
43
+
44
+ timeout = Datadog.configuration.appsec.waf_timeout
24
45
  result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_REQUEST_PHASE)
25
46
  handle(result, context: context) if result.match?
26
47
 
@@ -30,18 +51,50 @@ module Datadog
30
51
  private
31
52
 
32
53
  def on_complete(env, context:)
33
- timeout = Datadog.configuration.appsec.waf_timeout
34
-
35
- response_headers = env.response_headers || {}
54
+ headers = normalize_headers(env.response_headers)
55
+ # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
36
56
  ephemeral_data = {
37
57
  'server.io.net.response.status' => env.status.to_s,
38
- 'server.io.net.response.headers' => response_headers.transform_keys(&:downcase)
58
+ 'server.io.net.response.headers' => headers
39
59
  }
40
60
 
61
+ is_redirect = REDIRECT_STATUS_CODES.cover?(env.status) && headers.key?('location')
62
+ if is_redirect && env[SAMPLE_BODY_KEY]
63
+ context.state[:downstream_redirect_url] = URI.join(env.url.to_s, headers['location']).to_s
64
+ end
65
+
66
+ if !is_redirect && env[SAMPLE_BODY_KEY]
67
+ body = parse_body(env.body, content_type: headers['content-type'])
68
+ ephemeral_data['server.io.net.response.body'] = body if body
69
+ end
70
+
71
+ timeout = Datadog.configuration.appsec.waf_timeout
41
72
  result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_RESPONSE_PHASE)
42
73
  handle(result, context: context) if result.match?
43
74
  end
44
75
 
76
+ def mark_body_sampling!(env, context:)
77
+ max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_requests
78
+ return if context.state[:downstream_body_analyzed_count] >= max
79
+ return unless context.downstream_body_sampler.sample?
80
+
81
+ context.state[:downstream_body_analyzed_count] += 1
82
+ env[SAMPLE_BODY_KEY] = true
83
+ end
84
+
85
+ def parse_body(body, content_type:)
86
+ media_type = Utils::HTTP::MediaType.parse(content_type)
87
+ return unless media_type
88
+
89
+ Utils::HTTP::Body.parse(body, media_type: media_type)
90
+ end
91
+
92
+ def normalize_headers(headers)
93
+ return {} if headers.nil? || headers.empty?
94
+
95
+ headers.transform_keys(&:downcase)
96
+ end
97
+
45
98
  def handle(result, context:)
46
99
  AppSec::Event.tag(context, result)
47
100
  TraceKeeper.keep!(context.trace) if result.keep?
@@ -47,9 +47,10 @@ module Datadog
47
47
  # Note that the `comments` "include" directive is included in the arguments list
48
48
  def build_arguments_hash
49
49
  queries.each_with_object({}) do |query, args_hash|
50
- next unless query.selected_operation
50
+ selected_operation = query.selected_operation
51
+ next unless selected_operation
51
52
 
52
- arguments_from_selections(query.selected_operation.selections, query.variables, args_hash)
53
+ arguments_from_selections(selected_operation.selections, query.variables, args_hash)
53
54
  end
54
55
  end
55
56
 
@@ -90,15 +91,19 @@ module Datadog
90
91
  end
91
92
 
92
93
  def argument_value(argument, query_variables)
93
- case argument.value.class.name
94
+ value = argument.value
95
+
96
+ case value.class.name
94
97
  when Integration::AST_NODE_CLASS_NAMES[:variable_identifier]
98
+ # @type var value: GraphQL::Language::Nodes::VariableIdentifier
95
99
  # we need to pass query.variables here instead of query.provided_variables,
96
100
  # since #provided_variables don't know anything about variable default value
97
- query_variables[argument.value.name]
101
+ query_variables[value.name]
98
102
  when Integration::AST_NODE_CLASS_NAMES[:input_object]
99
- arguments_hash(argument.value.arguments, query_variables)
103
+ # @type var value: GraphQL::Language::Nodes::InputObject
104
+ arguments_hash(value.arguments, query_variables)
100
105
  else
101
- argument.value
106
+ value
102
107
  end
103
108
  end
104
109
  end
@@ -22,7 +22,7 @@ module Datadog
22
22
  end
23
23
 
24
24
  def watch_multiplex(gateway = Instrumentation.gateway)
25
- gateway.watch('graphql.multiplex', :appsec) do |stack, gateway_multiplex|
25
+ gateway.watch('graphql.multiplex') do |stack, gateway_multiplex|
26
26
  context = AppSec::Context.active
27
27
 
28
28
  if context
@@ -31,7 +31,7 @@ module Datadog
31
31
  end
32
32
 
33
33
  def self.compatible?
34
- super && version >= MINIMUM_VERSION && ast_node_classes_defined?
34
+ !!(super && version&.>=(MINIMUM_VERSION) && ast_node_classes_defined?)
35
35
  end
36
36
 
37
37
  def self.auto_instrument?
@@ -23,16 +23,12 @@ module Datadog
23
23
  end
24
24
 
25
25
  def query
26
- # Downstream libddwaf expects keys and values to be extractable
27
- # separately so we can't use [[k, v], ...]. We also want to allow
28
- # duplicate keys, so we use {k => [v, ...], ...} instead, taking into
29
- # account that {k => [v1, v2, ...], ...} is possible for duplicate keys.
30
- request.query_string.split('&').each.with_object({}) do |e, hash|
31
- k, v = e.split('=').map { |s| CGI.unescape(s) }
32
- hash[k] ||= []
33
-
34
- hash[k] << v
35
- end
26
+ ::Rack::Utils.parse_query(request.query_string)
27
+ rescue => e
28
+ Datadog.logger.debug { "AppSec: Failed to parse request query string: #{e.class}: #{e.message}" }
29
+ AppSec.telemetry.report(e, description: 'AppSec: Failed to parse request query string')
30
+
31
+ {}
36
32
  end
37
33
 
38
34
  def method
@@ -24,7 +24,7 @@ module Datadog
24
24
  end
25
25
 
26
26
  def watch_request(gateway = Instrumentation.gateway)
27
- gateway.watch('rack.request', :appsec) do |stack, gateway_request|
27
+ gateway.watch('rack.request') do |stack, gateway_request|
28
28
  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
29
29
 
30
30
  persistent_data = {
@@ -57,7 +57,7 @@ module Datadog
57
57
  end
58
58
 
59
59
  def watch_response(gateway = Instrumentation.gateway)
60
- gateway.watch('rack.response', :appsec) do |stack, gateway_response|
60
+ gateway.watch('rack.response') do |stack, gateway_response|
61
61
  context = gateway_response.context
62
62
 
63
63
  persistent_data = {
@@ -84,7 +84,7 @@ module Datadog
84
84
  end
85
85
 
86
86
  def watch_request_body(gateway = Instrumentation.gateway)
87
- gateway.watch('rack.request.body', :appsec) do |stack, gateway_request|
87
+ gateway.watch('rack.request.body') do |stack, gateway_request|
88
88
  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
89
89
 
90
90
  persistent_data = {
@@ -113,7 +113,7 @@ module Datadog
113
113
  # somewhere closer to identity related monitor.
114
114
  # WARNING: The Gateway is a subject of refactoring
115
115
  def watch_request_finish(gateway = Instrumentation.gateway)
116
- gateway.watch('rack.request.finish', :appsec) do |stack, gateway_request|
116
+ gateway.watch('rack.request.finish') do |stack, gateway_request|
117
117
  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
118
118
 
119
119
  if context.span.nil? || !gateway.pushed?('appsec.events.user_lifecycle')
@@ -27,7 +27,7 @@ module Datadog
27
27
  end
28
28
 
29
29
  def self.compatible?
30
- super && version >= MINIMUM_VERSION
30
+ !!(super && (version&.>= MINIMUM_VERSION))
31
31
  end
32
32
 
33
33
  def self.auto_instrument?
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'json'
4
4
 
5
+ require_relative 'response_body'
5
6
  require_relative 'gateway/request'
6
7
  require_relative 'gateway/response'
7
8
 
@@ -18,6 +19,13 @@ module Datadog
18
19
  module AppSec
19
20
  module Contrib
20
21
  module Rack
22
+ RESPONSE_HEADERS_TAGS = %w[
23
+ content-length
24
+ content-type
25
+ content-encoding
26
+ content-language
27
+ ].freeze
28
+
21
29
  WAF_VENDOR_HEADERS_TAGS = %w[
22
30
  X-Amzn-Trace-Id
23
31
  Cloudfront-Viewer-Ja3-Fingerprint
@@ -107,13 +115,12 @@ module Datadog
107
115
 
108
116
  if AppSec::APISecurity.enabled? && AppSec::APISecurity.sample_trace?(ctx.trace) &&
109
117
  AppSec::APISecurity.sample?(gateway_request.request, tmp_response.response)
110
- ctx.events.push(
111
- AppSec::SecurityEvent.new(ctx.extract_schema, trace: ctx.trace, span: ctx.span)
112
- )
118
+ ctx.extract_schema!
113
119
  end
114
120
 
115
- AppSec::Event.record(ctx, request: gateway_request, response: gateway_response)
121
+ AppSec::Event.record(ctx, request: gateway_request)
116
122
 
123
+ add_response_tags(ctx, tmp_response)
117
124
  http_response
118
125
  ensure
119
126
  if ctx
@@ -182,7 +189,6 @@ module Datadog
182
189
  # standard:disable Metrics/MethodLength
183
190
  def add_request_tags(context, env)
184
191
  span = context.span
185
-
186
192
  return unless span
187
193
 
188
194
  # Always add WAF vendors headers
@@ -204,6 +210,21 @@ module Datadog
204
210
  end
205
211
  # standard:enable Metrics/MethodLength
206
212
 
213
+ def add_response_tags(context, response)
214
+ span = context.span
215
+ return unless span
216
+
217
+ RESPONSE_HEADERS_TAGS.each do |name|
218
+ value = response.headers[name]
219
+ span.set_tag("http.response.headers.#{name}", value.to_s) if value
220
+ end
221
+
222
+ unless response.headers.key?('content-length')
223
+ length = ResponseBody.content_length(response.body)
224
+ span.set_tag('http.response.headers.content-length', length.to_s) if length
225
+ end
226
+ end
227
+
207
228
  def oneshot_tags_sent?
208
229
  @oneshot_tags_sent
209
230
  end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Datadog
4
+ module AppSec
5
+ module Contrib
6
+ module Rack
7
+ module ResponseBody
8
+ # NOTE: We compute content length only for fixed-size response bodies,
9
+ # ignoring streaming bodies to avoid buffering.
10
+ #
11
+ # On Rack 3.x, `body.to_ary` on a BodyProxy triggers `close` on all
12
+ # nested BodyProxy layers. This is safe because web servers, like Puma
13
+ # handles already-closed bodies (its own `to_ary` becomes a no-op).
14
+ #
15
+ # @see Puma::Response#prepare_response
16
+ # @see https://github.com/puma/puma/blob/b1271222cbf21868f3fb64154caa4d03936a7b9e/lib/puma/response.rb#L165-L168
17
+ def self.content_length(body)
18
+ return unless body.respond_to?(:to_ary)
19
+
20
+ # NOTE: When `to_ary` exists but returns `nil`, the body will be
21
+ # transferred in chunks and we can't compute content length
22
+ # without buffering it.
23
+ body_ary = body.to_ary
24
+ return unless body_ary.is_a?(Array)
25
+
26
+ body_ary.sum { |part| part.is_a?(String) ? part.bytesize : 0 }
27
+ rescue => e
28
+ AppSec.telemetry.report(e, description: 'AppSec: Failed to compute body content length')
29
+
30
+ nil
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -21,7 +21,7 @@ module Datadog
21
21
  end
22
22
 
23
23
  def watch_request_action(gateway = Instrumentation.gateway)
24
- gateway.watch('rails.request.action', :appsec) do |stack, gateway_request|
24
+ gateway.watch('rails.request.action') do |stack, gateway_request|
25
25
  context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
26
26
 
27
27
  persistent_data = {
@@ -47,7 +47,7 @@ module Datadog
47
47
  end
48
48
 
49
49
  def watch_response_body_json(gateway = Instrumentation.gateway)
50
- gateway.watch('rails.response.body.json', :appsec) do |stack, container|
50
+ gateway.watch('rails.response.body.json') do |stack, container|
51
51
  context = container.context
52
52
 
53
53
  persistent_data = {
@@ -26,7 +26,7 @@ module Datadog
26
26
  end
27
27
 
28
28
  def self.compatible?
29
- super && version >= MINIMUM_VERSION
29
+ !!(super && (version&.>= MINIMUM_VERSION))
30
30
  end
31
31
 
32
32
  def self.auto_instrument?