datadog 2.28.0 → 2.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +87 -1
- data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +82 -12
- data/ext/datadog_profiling_native_extension/collectors_thread_context.c +32 -11
- data/ext/datadog_profiling_native_extension/collectors_thread_context.h +3 -1
- data/ext/datadog_profiling_native_extension/extconf.rb +9 -24
- data/ext/datadog_profiling_native_extension/heap_recorder.c +186 -55
- data/ext/datadog_profiling_native_extension/heap_recorder.h +12 -1
- data/ext/datadog_profiling_native_extension/http_transport.c +51 -64
- data/ext/datadog_profiling_native_extension/native_extension_helpers.rb +0 -13
- data/ext/datadog_profiling_native_extension/profiling.c +3 -1
- data/ext/datadog_profiling_native_extension/setup_signal_handler.c +24 -8
- data/ext/datadog_profiling_native_extension/setup_signal_handler.h +1 -3
- data/ext/datadog_profiling_native_extension/stack_recorder.c +63 -48
- data/ext/datadog_profiling_native_extension/stack_recorder.h +2 -1
- data/ext/libdatadog_api/crashtracker.c +5 -0
- data/ext/libdatadog_api/crashtracker_report_exception.c +126 -0
- data/ext/libdatadog_api/extconf.rb +4 -21
- data/ext/libdatadog_extconf_helpers.rb +49 -11
- data/lib/datadog/ai_guard/configuration/settings.rb +3 -0
- data/lib/datadog/appsec/assets/blocked.html +2 -1
- data/lib/datadog/appsec/configuration/settings.rb +14 -0
- data/lib/datadog/appsec/context.rb +44 -9
- data/lib/datadog/appsec/contrib/active_record/patcher.rb +3 -0
- data/lib/datadog/appsec/contrib/devise/integration.rb +1 -1
- data/lib/datadog/appsec/contrib/excon/patcher.rb +2 -0
- data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb +55 -6
- data/lib/datadog/appsec/contrib/faraday/integration.rb +1 -1
- data/lib/datadog/appsec/contrib/faraday/patcher.rb +1 -1
- data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb +60 -7
- data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb +11 -6
- data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb +1 -1
- data/lib/datadog/appsec/contrib/graphql/integration.rb +1 -1
- data/lib/datadog/appsec/contrib/rack/gateway/request.rb +6 -10
- data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +4 -4
- data/lib/datadog/appsec/contrib/rack/integration.rb +1 -1
- data/lib/datadog/appsec/contrib/rack/request_middleware.rb +26 -5
- data/lib/datadog/appsec/contrib/rack/response_body.rb +36 -0
- data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +2 -2
- data/lib/datadog/appsec/contrib/rails/integration.rb +1 -1
- data/lib/datadog/appsec/contrib/rails/patches/process_action_patch.rb +2 -0
- data/lib/datadog/appsec/contrib/rest_client/patcher.rb +2 -0
- data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb +72 -7
- data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +5 -3
- data/lib/datadog/appsec/counter_sampler.rb +25 -0
- data/lib/datadog/appsec/event.rb +1 -17
- data/lib/datadog/appsec/instrumentation/gateway/middleware.rb +2 -3
- data/lib/datadog/appsec/instrumentation/gateway.rb +2 -2
- data/lib/datadog/appsec/metrics/telemetry_exporter.rb +18 -0
- data/lib/datadog/appsec/monitor/gateway/watcher.rb +2 -2
- data/lib/datadog/appsec/security_engine/engine.rb +23 -2
- data/lib/datadog/appsec/utils/http/body.rb +38 -0
- data/lib/datadog/appsec/utils/http/media_range.rb +2 -1
- data/lib/datadog/appsec/utils/http/media_type.rb +28 -35
- data/lib/datadog/appsec/utils/http/url_encoded.rb +52 -0
- data/lib/datadog/core/configuration/components.rb +29 -4
- data/lib/datadog/core/configuration/option.rb +2 -1
- data/lib/datadog/core/configuration/options.rb +1 -1
- data/lib/datadog/core/configuration/settings.rb +27 -3
- data/lib/datadog/core/configuration/supported_configurations.rb +19 -0
- data/lib/datadog/core/configuration.rb +2 -2
- data/lib/datadog/core/crashtracking/component.rb +71 -19
- data/lib/datadog/core/environment/agent_info.rb +65 -1
- data/lib/datadog/core/logger.rb +1 -1
- data/lib/datadog/core/metrics/logging.rb +1 -1
- data/lib/datadog/core/process_discovery.rb +20 -19
- data/lib/datadog/core/rate_limiter.rb +2 -0
- data/lib/datadog/core/remote/component.rb +16 -5
- data/lib/datadog/core/remote/transport/config.rb +5 -11
- data/lib/datadog/core/runtime/metrics.rb +1 -2
- data/lib/datadog/core/telemetry/component.rb +0 -13
- data/lib/datadog/core/telemetry/transport/telemetry.rb +5 -6
- data/lib/datadog/core/transport/ext.rb +1 -0
- data/lib/datadog/core/transport/http/response.rb +4 -0
- data/lib/datadog/core/transport/parcel.rb +61 -9
- data/lib/datadog/core/utils/base64.rb +1 -1
- data/lib/datadog/core/utils/fnv.rb +26 -0
- data/lib/datadog/core/workers/interval_loop.rb +13 -6
- data/lib/datadog/core/workers/queue.rb +0 -4
- data/lib/datadog/core/workers/runtime_metrics.rb +9 -1
- data/lib/datadog/core.rb +6 -1
- data/lib/datadog/data_streams/processor.rb +35 -33
- data/lib/datadog/data_streams/transport/http/stats.rb +6 -0
- data/lib/datadog/data_streams/transport/http.rb +0 -4
- data/lib/datadog/data_streams/transport/stats.rb +5 -12
- data/lib/datadog/di/boot.rb +1 -0
- data/lib/datadog/di/component.rb +17 -5
- data/lib/datadog/di/configuration/settings.rb +9 -0
- data/lib/datadog/di/context.rb +6 -0
- data/lib/datadog/di/instrumenter.rb +183 -134
- data/lib/datadog/di/probe.rb +10 -1
- data/lib/datadog/di/probe_file_loader.rb +2 -2
- data/lib/datadog/di/probe_manager.rb +86 -64
- data/lib/datadog/di/probe_notification_builder.rb +46 -18
- data/lib/datadog/di/probe_notifier_worker.rb +65 -9
- data/lib/datadog/di/probe_repository.rb +198 -0
- data/lib/datadog/di/proc_responder.rb +4 -0
- data/lib/datadog/di/remote.rb +6 -7
- data/lib/datadog/di/serializer.rb +127 -9
- data/lib/datadog/di/transport/diagnostics.rb +5 -7
- data/lib/datadog/di/transport/http/diagnostics.rb +3 -1
- data/lib/datadog/di/transport/http/input.rb +1 -1
- data/lib/datadog/di/transport/http.rb +12 -3
- data/lib/datadog/di/transport/input.rb +51 -14
- data/lib/datadog/kit/tracing/method_tracer.rb +132 -0
- data/lib/datadog/open_feature/configuration.rb +2 -0
- data/lib/datadog/open_feature/transport.rb +8 -11
- data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +13 -0
- data/lib/datadog/profiling/component.rb +20 -6
- data/lib/datadog/profiling/http_transport.rb +5 -6
- data/lib/datadog/profiling/profiler.rb +15 -8
- data/lib/datadog/tracing/contrib/dalli/integration.rb +4 -1
- data/lib/datadog/tracing/contrib/ethon/configuration/settings.rb +5 -1
- data/lib/datadog/tracing/contrib/ethon/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/excon/configuration/settings.rb +5 -2
- data/lib/datadog/tracing/contrib/excon/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/faraday/configuration/settings.rb +5 -2
- data/lib/datadog/tracing/contrib/faraday/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/grape/endpoint.rb +2 -2
- data/lib/datadog/tracing/contrib/grape/instrumentation.rb +13 -8
- data/lib/datadog/tracing/contrib/grape/patcher.rb +6 -1
- data/lib/datadog/tracing/contrib/grpc/configuration/settings.rb +5 -2
- data/lib/datadog/tracing/contrib/grpc/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/http/configuration/settings.rb +5 -2
- data/lib/datadog/tracing/contrib/http/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/http/integration.rb +0 -2
- data/lib/datadog/tracing/contrib/httpclient/configuration/settings.rb +5 -2
- data/lib/datadog/tracing/contrib/httpclient/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/httprb/configuration/settings.rb +5 -2
- data/lib/datadog/tracing/contrib/httprb/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/karafka/configuration/settings.rb +5 -1
- data/lib/datadog/tracing/contrib/karafka/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/mysql2/configuration/settings.rb +6 -0
- data/lib/datadog/tracing/contrib/mysql2/instrumentation.rb +2 -1
- data/lib/datadog/tracing/contrib/pg/configuration/settings.rb +6 -0
- data/lib/datadog/tracing/contrib/pg/instrumentation.rb +2 -1
- data/lib/datadog/tracing/contrib/propagation/sql_comment/ext.rb +10 -0
- data/lib/datadog/tracing/contrib/propagation/sql_comment/mode.rb +5 -1
- data/lib/datadog/tracing/contrib/propagation/sql_comment.rb +24 -0
- data/lib/datadog/tracing/contrib/que/configuration/settings.rb +5 -2
- data/lib/datadog/tracing/contrib/que/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/rack/configuration/settings.rb +5 -1
- data/lib/datadog/tracing/contrib/rack/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/rack/route_inference.rb +18 -6
- data/lib/datadog/tracing/contrib/rails/configuration/settings.rb +5 -2
- data/lib/datadog/tracing/contrib/rails/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/registerable.rb +11 -0
- data/lib/datadog/tracing/contrib/rest_client/configuration/settings.rb +5 -2
- data/lib/datadog/tracing/contrib/rest_client/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/sidekiq/configuration/settings.rb +5 -1
- data/lib/datadog/tracing/contrib/sidekiq/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/sinatra/configuration/settings.rb +5 -1
- data/lib/datadog/tracing/contrib/sinatra/ext.rb +1 -0
- data/lib/datadog/tracing/contrib/sneakers/integration.rb +15 -4
- data/lib/datadog/tracing/contrib/trilogy/configuration/settings.rb +6 -0
- data/lib/datadog/tracing/contrib/trilogy/instrumentation.rb +3 -1
- data/lib/datadog/tracing/contrib/waterdrop/configuration/settings.rb +5 -1
- data/lib/datadog/tracing/contrib/waterdrop/ext.rb +1 -0
- data/lib/datadog/tracing/metadata/ext.rb +4 -0
- data/lib/datadog/tracing/sync_writer.rb +0 -1
- data/lib/datadog/tracing/transport/io/client.rb +5 -8
- data/lib/datadog/tracing/transport/io/traces.rb +28 -34
- data/lib/datadog/tracing/transport/trace_formatter.rb +11 -0
- data/lib/datadog/tracing/transport/traces.rb +4 -10
- data/lib/datadog/tracing/writer.rb +0 -1
- data/lib/datadog/version.rb +1 -1
- metadata +14 -8
- data/lib/datadog/appsec/contrib/rails/ext.rb +0 -13
- data/lib/datadog/tracing/workers/trace_writer.rb +0 -204
|
@@ -100,7 +100,8 @@
|
|
|
100
100
|
<footer>
|
|
101
101
|
<p>Security provided by <a
|
|
102
102
|
href="https://www.datadoghq.com/product/security-platform/application-security-monitoring/"
|
|
103
|
-
target="_blank"
|
|
103
|
+
target="_blank"
|
|
104
|
+
rel="noopener noreferrer">Datadog</a></p>
|
|
104
105
|
</footer>
|
|
105
106
|
</body>
|
|
106
107
|
|
|
@@ -393,6 +393,20 @@ module Datadog
|
|
|
393
393
|
end
|
|
394
394
|
end
|
|
395
395
|
end
|
|
396
|
+
|
|
397
|
+
settings :downstream_body_analysis do
|
|
398
|
+
option :sample_rate do |o|
|
|
399
|
+
o.type :float
|
|
400
|
+
o.env 'DD_API_SECURITY_DOWNSTREAM_BODY_ANALYSIS_SAMPLE_RATE'
|
|
401
|
+
o.default 0.5
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
option :max_requests do |o|
|
|
405
|
+
o.type :int
|
|
406
|
+
o.env 'DD_API_SECURITY_MAX_DOWNSTREAM_REQUEST_BODY_ANALYSIS'
|
|
407
|
+
o.default 1
|
|
408
|
+
end
|
|
409
|
+
end
|
|
396
410
|
end
|
|
397
411
|
|
|
398
412
|
option :sca_enabled do |o|
|
|
@@ -1,17 +1,36 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative 'counter_sampler'
|
|
3
4
|
require_relative 'metrics'
|
|
5
|
+
require_relative 'security_event'
|
|
4
6
|
|
|
5
7
|
module Datadog
|
|
6
8
|
module AppSec
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
+
# Request-bound context providing threat detection interface.
|
|
10
|
+
#
|
|
11
|
+
# Activated at the start of a request (see `Contrib::Rack::RequestMiddleware`)
|
|
12
|
+
# and shared across all instrumentations within that request's lifecycle.
|
|
13
|
+
#
|
|
14
|
+
# Accumulates security events, metrics, and state needed for coordinated
|
|
15
|
+
# threat detection.
|
|
16
|
+
#
|
|
17
|
+
# @api private
|
|
9
18
|
class Context
|
|
10
19
|
# Steep: https://github.com/soutaro/steep/issues/1880
|
|
11
20
|
ActiveContextError = Class.new(StandardError) # steep:ignore IncompatibleAssignment
|
|
12
21
|
|
|
13
22
|
# TODO: add delegators for active trace span
|
|
14
|
-
attr_reader :trace, :span
|
|
23
|
+
attr_reader :trace, :span
|
|
24
|
+
|
|
25
|
+
# Shared mutable storage for counters, flags, and data accumulated during
|
|
26
|
+
# the request's lifecycle.
|
|
27
|
+
#
|
|
28
|
+
# NOTE: This attribute is a subject to change, but in a current form
|
|
29
|
+
# it's a `Hash`-like structure.
|
|
30
|
+
attr_reader :state
|
|
31
|
+
|
|
32
|
+
# Sampler for downstream HTTP request/response body analysis.
|
|
33
|
+
attr_reader :downstream_body_sampler
|
|
15
34
|
|
|
16
35
|
class << self
|
|
17
36
|
def activate(context)
|
|
@@ -35,10 +54,16 @@ module Datadog
|
|
|
35
54
|
def initialize(trace, span, waf_runner)
|
|
36
55
|
@trace = trace
|
|
37
56
|
@span = span
|
|
38
|
-
@events = []
|
|
39
57
|
@waf_runner = waf_runner
|
|
40
58
|
@metrics = Metrics::Collector.new
|
|
41
|
-
@
|
|
59
|
+
@downstream_body_sampler = CounterSampler.new(
|
|
60
|
+
Datadog.configuration.appsec.api_security.downstream_body_analysis.sample_rate
|
|
61
|
+
)
|
|
62
|
+
@state = {
|
|
63
|
+
events: [],
|
|
64
|
+
interrupted: false,
|
|
65
|
+
downstream_body_analyzed_count: 0
|
|
66
|
+
}
|
|
42
67
|
end
|
|
43
68
|
|
|
44
69
|
def run_waf(persistent_data, ephemeral_data, timeout = WAF::LibDDWAF::DDWAF_RUN_TIMEOUT)
|
|
@@ -57,12 +82,16 @@ module Datadog
|
|
|
57
82
|
result
|
|
58
83
|
end
|
|
59
84
|
|
|
85
|
+
def events
|
|
86
|
+
@state[:events]
|
|
87
|
+
end
|
|
88
|
+
|
|
60
89
|
def mark_as_interrupted!
|
|
61
|
-
@interrupted = true
|
|
90
|
+
@state[:interrupted] = true
|
|
62
91
|
end
|
|
63
92
|
|
|
64
93
|
def interrupted?
|
|
65
|
-
@interrupted
|
|
94
|
+
@state[:interrupted]
|
|
66
95
|
end
|
|
67
96
|
|
|
68
97
|
def waf_runner_ruleset_version
|
|
@@ -73,8 +102,13 @@ module Datadog
|
|
|
73
102
|
@waf_runner.waf_addresses
|
|
74
103
|
end
|
|
75
104
|
|
|
76
|
-
def extract_schema
|
|
77
|
-
@waf_runner.run({'waf.context.processor' => {'extract-schema' => true}}, {})
|
|
105
|
+
def extract_schema!
|
|
106
|
+
waf_result = @waf_runner.run({'waf.context.processor' => {'extract-schema' => true}}, {})
|
|
107
|
+
security_event = AppSec::SecurityEvent.new(waf_result, trace: trace, span: span)
|
|
108
|
+
|
|
109
|
+
@state[:schema_extracted] = security_event.schema?
|
|
110
|
+
|
|
111
|
+
events.push(security_event)
|
|
78
112
|
end
|
|
79
113
|
|
|
80
114
|
def export_metrics
|
|
@@ -88,6 +122,7 @@ module Datadog
|
|
|
88
122
|
return if @trace.nil?
|
|
89
123
|
|
|
90
124
|
Metrics::TelemetryExporter.export_waf_request_metrics(@metrics.waf, self)
|
|
125
|
+
Metrics::TelemetryExporter.export_api_security_metrics(self)
|
|
91
126
|
end
|
|
92
127
|
|
|
93
128
|
def finalize!
|
|
@@ -61,6 +61,7 @@ module Datadog
|
|
|
61
61
|
end
|
|
62
62
|
|
|
63
63
|
::ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend(instrumentation_module)
|
|
64
|
+
Patcher.instance_variable_set(:@patched, true)
|
|
64
65
|
end
|
|
65
66
|
|
|
66
67
|
def patch_mysql2_adapter
|
|
@@ -73,6 +74,7 @@ module Datadog
|
|
|
73
74
|
end
|
|
74
75
|
|
|
75
76
|
::ActiveRecord::ConnectionAdapters::Mysql2Adapter.prepend(instrumentation_module)
|
|
77
|
+
Patcher.instance_variable_set(:@patched, true)
|
|
76
78
|
end
|
|
77
79
|
|
|
78
80
|
def patch_postgresql_adapter
|
|
@@ -93,6 +95,7 @@ module Datadog
|
|
|
93
95
|
end
|
|
94
96
|
|
|
95
97
|
::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(instrumentation_module)
|
|
98
|
+
Patcher.instance_variable_set(:@patched, true)
|
|
96
99
|
end
|
|
97
100
|
end
|
|
98
101
|
end
|
|
@@ -5,6 +5,8 @@ require 'excon'
|
|
|
5
5
|
require_relative '../../event'
|
|
6
6
|
require_relative '../../trace_keeper'
|
|
7
7
|
require_relative '../../security_event'
|
|
8
|
+
require_relative '../../utils/http/url_encoded'
|
|
9
|
+
require_relative '../../utils/http/body'
|
|
8
10
|
|
|
9
11
|
module Datadog
|
|
10
12
|
module AppSec
|
|
@@ -12,17 +14,36 @@ module Datadog
|
|
|
12
14
|
module Excon
|
|
13
15
|
# AppSec Middleware for Excon
|
|
14
16
|
class SSRFDetectionMiddleware < ::Excon::Middleware::Base
|
|
17
|
+
REDIRECT_STATUS_CODES = (300..399).freeze
|
|
18
|
+
SAMPLE_BODY_KEY = :__datadog_appsec_sample_downstream_body
|
|
19
|
+
|
|
15
20
|
def request_call(data)
|
|
16
21
|
context = AppSec.active_context
|
|
17
22
|
return super unless context && AppSec.rasp_enabled?
|
|
18
23
|
|
|
19
|
-
|
|
24
|
+
url = request_url(data)
|
|
25
|
+
headers = normalize_headers(data[:headers])
|
|
26
|
+
# @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
|
|
20
27
|
ephemeral_data = {
|
|
21
|
-
'server.io.net.url' =>
|
|
28
|
+
'server.io.net.url' => url,
|
|
22
29
|
'server.io.net.request.method' => data[:method].to_s.upcase,
|
|
23
|
-
'server.io.net.request.headers' =>
|
|
30
|
+
'server.io.net.request.headers' => headers
|
|
24
31
|
}
|
|
25
32
|
|
|
33
|
+
is_redirect = context.state[:downstream_redirect_url] == url
|
|
34
|
+
if is_redirect
|
|
35
|
+
context.state.delete(:downstream_redirect_url)
|
|
36
|
+
data[SAMPLE_BODY_KEY] = true
|
|
37
|
+
else
|
|
38
|
+
mark_body_sampling!(data, context: context)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
if !is_redirect && data[SAMPLE_BODY_KEY]
|
|
42
|
+
body = parse_body(data[:body], content_type: headers['content-type'])
|
|
43
|
+
ephemeral_data['server.io.net.request.body'] = body if body
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
timeout = Datadog.configuration.appsec.waf_timeout
|
|
26
47
|
result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_REQUEST_PHASE)
|
|
27
48
|
handle(result, context: context) if result.match?
|
|
28
49
|
|
|
@@ -33,12 +54,24 @@ module Datadog
|
|
|
33
54
|
context = AppSec.active_context
|
|
34
55
|
return super unless context && AppSec.rasp_enabled?
|
|
35
56
|
|
|
36
|
-
|
|
57
|
+
headers = normalize_headers(data.dig(:response, :headers))
|
|
58
|
+
# @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
|
|
37
59
|
ephemeral_data = {
|
|
38
60
|
'server.io.net.response.status' => data.dig(:response, :status).to_s,
|
|
39
|
-
'server.io.net.response.headers' =>
|
|
61
|
+
'server.io.net.response.headers' => headers
|
|
40
62
|
}
|
|
41
63
|
|
|
64
|
+
is_redirect = REDIRECT_STATUS_CODES.cover?(data.dig(:response, :status)) && headers.key?('location')
|
|
65
|
+
if is_redirect && data[SAMPLE_BODY_KEY]
|
|
66
|
+
context.state[:downstream_redirect_url] = URI.join(request_url(data), headers['location']).to_s
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
if !is_redirect && data[SAMPLE_BODY_KEY]
|
|
70
|
+
body = parse_body(data.dig(:response, :body), content_type: headers['content-type'])
|
|
71
|
+
ephemeral_data['server.io.net.response.body'] = body if body
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
timeout = Datadog.configuration.appsec.waf_timeout
|
|
42
75
|
result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_RESPONSE_PHASE)
|
|
43
76
|
handle(result, context: context) if result.match?
|
|
44
77
|
|
|
@@ -47,9 +80,25 @@ module Datadog
|
|
|
47
80
|
|
|
48
81
|
private
|
|
49
82
|
|
|
83
|
+
def mark_body_sampling!(data, context:)
|
|
84
|
+
max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_requests
|
|
85
|
+
return if context.state[:downstream_body_analyzed_count] >= max
|
|
86
|
+
return unless context.downstream_body_sampler.sample?
|
|
87
|
+
|
|
88
|
+
context.state[:downstream_body_analyzed_count] += 1
|
|
89
|
+
data[SAMPLE_BODY_KEY] = true
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def parse_body(body, content_type:)
|
|
93
|
+
media_type = Utils::HTTP::MediaType.parse(content_type)
|
|
94
|
+
return unless media_type
|
|
95
|
+
|
|
96
|
+
Utils::HTTP::Body.parse(body, media_type: media_type)
|
|
97
|
+
end
|
|
98
|
+
|
|
50
99
|
def request_url(data)
|
|
51
100
|
klass = (data[:scheme] == 'https') ? URI::HTTPS : URI::HTTP
|
|
52
|
-
klass.build(host: data[:host], path: data[:path], query: data[:query]).to_s
|
|
101
|
+
klass.build(host: data[:host], port: data[:port], path: data[:path], query: data[:query]).to_s
|
|
53
102
|
end
|
|
54
103
|
|
|
55
104
|
def normalize_headers(headers)
|
|
@@ -28,7 +28,7 @@ module Datadog
|
|
|
28
28
|
end
|
|
29
29
|
|
|
30
30
|
def configure_default_faraday_connection
|
|
31
|
-
if target_version
|
|
31
|
+
if target_version&.>= Gem::Version.new('1.0.0')
|
|
32
32
|
# Patch the default connection (e.g. +Faraday.get+)
|
|
33
33
|
::Faraday.default_connection.use(:datadog_appsec)
|
|
34
34
|
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
require_relative '../../event'
|
|
4
4
|
require_relative '../../trace_keeper'
|
|
5
5
|
require_relative '../../security_event'
|
|
6
|
+
require_relative '../../utils/http/media_type'
|
|
7
|
+
require_relative '../../utils/http/body'
|
|
6
8
|
|
|
7
9
|
module Datadog
|
|
8
10
|
module AppSec
|
|
@@ -10,17 +12,36 @@ module Datadog
|
|
|
10
12
|
module Faraday
|
|
11
13
|
# AppSec SSRF detection Middleware for Faraday
|
|
12
14
|
class SSRFDetectionMiddleware < ::Faraday::Middleware
|
|
15
|
+
REDIRECT_STATUS_CODES = (300..399).freeze
|
|
16
|
+
SAMPLE_BODY_KEY = :__datadog_appsec_sample_downstream_body
|
|
17
|
+
|
|
13
18
|
def call(env)
|
|
14
19
|
context = AppSec.active_context
|
|
15
20
|
return @app.call(env) unless context && AppSec.rasp_enabled?
|
|
16
21
|
|
|
17
|
-
|
|
22
|
+
url = env.url.to_s
|
|
23
|
+
headers = normalize_headers(env.request_headers)
|
|
24
|
+
# @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
|
|
18
25
|
ephemeral_data = {
|
|
19
|
-
'server.io.net.url' =>
|
|
26
|
+
'server.io.net.url' => url,
|
|
20
27
|
'server.io.net.request.method' => env.method.to_s.upcase,
|
|
21
|
-
'server.io.net.request.headers' =>
|
|
28
|
+
'server.io.net.request.headers' => headers
|
|
22
29
|
}
|
|
23
30
|
|
|
31
|
+
is_redirect = context.state[:downstream_redirect_url] == url
|
|
32
|
+
if is_redirect
|
|
33
|
+
context.state.delete(:downstream_redirect_url)
|
|
34
|
+
env[SAMPLE_BODY_KEY] = true
|
|
35
|
+
else
|
|
36
|
+
mark_body_sampling!(env, context: context)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
if !is_redirect && env[SAMPLE_BODY_KEY]
|
|
40
|
+
body = parse_body(env.body, content_type: headers['content-type'])
|
|
41
|
+
ephemeral_data['server.io.net.request.body'] = body if body
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
timeout = Datadog.configuration.appsec.waf_timeout
|
|
24
45
|
result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_REQUEST_PHASE)
|
|
25
46
|
handle(result, context: context) if result.match?
|
|
26
47
|
|
|
@@ -30,18 +51,50 @@ module Datadog
|
|
|
30
51
|
private
|
|
31
52
|
|
|
32
53
|
def on_complete(env, context:)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
response_headers = env.response_headers || {}
|
|
54
|
+
headers = normalize_headers(env.response_headers)
|
|
55
|
+
# @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
|
|
36
56
|
ephemeral_data = {
|
|
37
57
|
'server.io.net.response.status' => env.status.to_s,
|
|
38
|
-
'server.io.net.response.headers' =>
|
|
58
|
+
'server.io.net.response.headers' => headers
|
|
39
59
|
}
|
|
40
60
|
|
|
61
|
+
is_redirect = REDIRECT_STATUS_CODES.cover?(env.status) && headers.key?('location')
|
|
62
|
+
if is_redirect && env[SAMPLE_BODY_KEY]
|
|
63
|
+
context.state[:downstream_redirect_url] = URI.join(env.url.to_s, headers['location']).to_s
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
if !is_redirect && env[SAMPLE_BODY_KEY]
|
|
67
|
+
body = parse_body(env.body, content_type: headers['content-type'])
|
|
68
|
+
ephemeral_data['server.io.net.response.body'] = body if body
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
timeout = Datadog.configuration.appsec.waf_timeout
|
|
41
72
|
result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, timeout, phase: Ext::RASP_RESPONSE_PHASE)
|
|
42
73
|
handle(result, context: context) if result.match?
|
|
43
74
|
end
|
|
44
75
|
|
|
76
|
+
def mark_body_sampling!(env, context:)
|
|
77
|
+
max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_requests
|
|
78
|
+
return if context.state[:downstream_body_analyzed_count] >= max
|
|
79
|
+
return unless context.downstream_body_sampler.sample?
|
|
80
|
+
|
|
81
|
+
context.state[:downstream_body_analyzed_count] += 1
|
|
82
|
+
env[SAMPLE_BODY_KEY] = true
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def parse_body(body, content_type:)
|
|
86
|
+
media_type = Utils::HTTP::MediaType.parse(content_type)
|
|
87
|
+
return unless media_type
|
|
88
|
+
|
|
89
|
+
Utils::HTTP::Body.parse(body, media_type: media_type)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def normalize_headers(headers)
|
|
93
|
+
return {} if headers.nil? || headers.empty?
|
|
94
|
+
|
|
95
|
+
headers.transform_keys(&:downcase)
|
|
96
|
+
end
|
|
97
|
+
|
|
45
98
|
def handle(result, context:)
|
|
46
99
|
AppSec::Event.tag(context, result)
|
|
47
100
|
TraceKeeper.keep!(context.trace) if result.keep?
|
|
@@ -47,9 +47,10 @@ module Datadog
|
|
|
47
47
|
# Note that the `comments` "include" directive is included in the arguments list
|
|
48
48
|
def build_arguments_hash
|
|
49
49
|
queries.each_with_object({}) do |query, args_hash|
|
|
50
|
-
|
|
50
|
+
selected_operation = query.selected_operation
|
|
51
|
+
next unless selected_operation
|
|
51
52
|
|
|
52
|
-
arguments_from_selections(
|
|
53
|
+
arguments_from_selections(selected_operation.selections, query.variables, args_hash)
|
|
53
54
|
end
|
|
54
55
|
end
|
|
55
56
|
|
|
@@ -90,15 +91,19 @@ module Datadog
|
|
|
90
91
|
end
|
|
91
92
|
|
|
92
93
|
def argument_value(argument, query_variables)
|
|
93
|
-
|
|
94
|
+
value = argument.value
|
|
95
|
+
|
|
96
|
+
case value.class.name
|
|
94
97
|
when Integration::AST_NODE_CLASS_NAMES[:variable_identifier]
|
|
98
|
+
# @type var value: GraphQL::Language::Nodes::VariableIdentifier
|
|
95
99
|
# we need to pass query.variables here instead of query.provided_variables,
|
|
96
100
|
# since #provided_variables don't know anything about variable default value
|
|
97
|
-
query_variables[
|
|
101
|
+
query_variables[value.name]
|
|
98
102
|
when Integration::AST_NODE_CLASS_NAMES[:input_object]
|
|
99
|
-
|
|
103
|
+
# @type var value: GraphQL::Language::Nodes::InputObject
|
|
104
|
+
arguments_hash(value.arguments, query_variables)
|
|
100
105
|
else
|
|
101
|
-
|
|
106
|
+
value
|
|
102
107
|
end
|
|
103
108
|
end
|
|
104
109
|
end
|
|
@@ -22,7 +22,7 @@ module Datadog
|
|
|
22
22
|
end
|
|
23
23
|
|
|
24
24
|
def watch_multiplex(gateway = Instrumentation.gateway)
|
|
25
|
-
gateway.watch('graphql.multiplex'
|
|
25
|
+
gateway.watch('graphql.multiplex') do |stack, gateway_multiplex|
|
|
26
26
|
context = AppSec::Context.active
|
|
27
27
|
|
|
28
28
|
if context
|
|
@@ -23,16 +23,12 @@ module Datadog
|
|
|
23
23
|
end
|
|
24
24
|
|
|
25
25
|
def query
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
hash[k] ||= []
|
|
33
|
-
|
|
34
|
-
hash[k] << v
|
|
35
|
-
end
|
|
26
|
+
::Rack::Utils.parse_query(request.query_string)
|
|
27
|
+
rescue => e
|
|
28
|
+
Datadog.logger.debug { "AppSec: Failed to parse request query string: #{e.class}: #{e.message}" }
|
|
29
|
+
AppSec.telemetry.report(e, description: 'AppSec: Failed to parse request query string')
|
|
30
|
+
|
|
31
|
+
{}
|
|
36
32
|
end
|
|
37
33
|
|
|
38
34
|
def method
|
|
@@ -24,7 +24,7 @@ module Datadog
|
|
|
24
24
|
end
|
|
25
25
|
|
|
26
26
|
def watch_request(gateway = Instrumentation.gateway)
|
|
27
|
-
gateway.watch('rack.request'
|
|
27
|
+
gateway.watch('rack.request') do |stack, gateway_request|
|
|
28
28
|
context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
|
|
29
29
|
|
|
30
30
|
persistent_data = {
|
|
@@ -57,7 +57,7 @@ module Datadog
|
|
|
57
57
|
end
|
|
58
58
|
|
|
59
59
|
def watch_response(gateway = Instrumentation.gateway)
|
|
60
|
-
gateway.watch('rack.response'
|
|
60
|
+
gateway.watch('rack.response') do |stack, gateway_response|
|
|
61
61
|
context = gateway_response.context
|
|
62
62
|
|
|
63
63
|
persistent_data = {
|
|
@@ -84,7 +84,7 @@ module Datadog
|
|
|
84
84
|
end
|
|
85
85
|
|
|
86
86
|
def watch_request_body(gateway = Instrumentation.gateway)
|
|
87
|
-
gateway.watch('rack.request.body'
|
|
87
|
+
gateway.watch('rack.request.body') do |stack, gateway_request|
|
|
88
88
|
context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
|
|
89
89
|
|
|
90
90
|
persistent_data = {
|
|
@@ -113,7 +113,7 @@ module Datadog
|
|
|
113
113
|
# somewhere closer to identity related monitor.
|
|
114
114
|
# WARNING: The Gateway is a subject of refactoring
|
|
115
115
|
def watch_request_finish(gateway = Instrumentation.gateway)
|
|
116
|
-
gateway.watch('rack.request.finish'
|
|
116
|
+
gateway.watch('rack.request.finish') do |stack, gateway_request|
|
|
117
117
|
context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
|
|
118
118
|
|
|
119
119
|
if context.span.nil? || !gateway.pushed?('appsec.events.user_lifecycle')
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require 'json'
|
|
4
4
|
|
|
5
|
+
require_relative 'response_body'
|
|
5
6
|
require_relative 'gateway/request'
|
|
6
7
|
require_relative 'gateway/response'
|
|
7
8
|
|
|
@@ -18,6 +19,13 @@ module Datadog
|
|
|
18
19
|
module AppSec
|
|
19
20
|
module Contrib
|
|
20
21
|
module Rack
|
|
22
|
+
RESPONSE_HEADERS_TAGS = %w[
|
|
23
|
+
content-length
|
|
24
|
+
content-type
|
|
25
|
+
content-encoding
|
|
26
|
+
content-language
|
|
27
|
+
].freeze
|
|
28
|
+
|
|
21
29
|
WAF_VENDOR_HEADERS_TAGS = %w[
|
|
22
30
|
X-Amzn-Trace-Id
|
|
23
31
|
Cloudfront-Viewer-Ja3-Fingerprint
|
|
@@ -107,13 +115,12 @@ module Datadog
|
|
|
107
115
|
|
|
108
116
|
if AppSec::APISecurity.enabled? && AppSec::APISecurity.sample_trace?(ctx.trace) &&
|
|
109
117
|
AppSec::APISecurity.sample?(gateway_request.request, tmp_response.response)
|
|
110
|
-
ctx.
|
|
111
|
-
AppSec::SecurityEvent.new(ctx.extract_schema, trace: ctx.trace, span: ctx.span)
|
|
112
|
-
)
|
|
118
|
+
ctx.extract_schema!
|
|
113
119
|
end
|
|
114
120
|
|
|
115
|
-
AppSec::Event.record(ctx, request: gateway_request
|
|
121
|
+
AppSec::Event.record(ctx, request: gateway_request)
|
|
116
122
|
|
|
123
|
+
add_response_tags(ctx, tmp_response)
|
|
117
124
|
http_response
|
|
118
125
|
ensure
|
|
119
126
|
if ctx
|
|
@@ -182,7 +189,6 @@ module Datadog
|
|
|
182
189
|
# standard:disable Metrics/MethodLength
|
|
183
190
|
def add_request_tags(context, env)
|
|
184
191
|
span = context.span
|
|
185
|
-
|
|
186
192
|
return unless span
|
|
187
193
|
|
|
188
194
|
# Always add WAF vendors headers
|
|
@@ -204,6 +210,21 @@ module Datadog
|
|
|
204
210
|
end
|
|
205
211
|
# standard:enable Metrics/MethodLength
|
|
206
212
|
|
|
213
|
+
def add_response_tags(context, response)
|
|
214
|
+
span = context.span
|
|
215
|
+
return unless span
|
|
216
|
+
|
|
217
|
+
RESPONSE_HEADERS_TAGS.each do |name|
|
|
218
|
+
value = response.headers[name]
|
|
219
|
+
span.set_tag("http.response.headers.#{name}", value.to_s) if value
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
unless response.headers.key?('content-length')
|
|
223
|
+
length = ResponseBody.content_length(response.body)
|
|
224
|
+
span.set_tag('http.response.headers.content-length', length.to_s) if length
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
|
|
207
228
|
def oneshot_tags_sent?
|
|
208
229
|
@oneshot_tags_sent
|
|
209
230
|
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Datadog
|
|
4
|
+
module AppSec
|
|
5
|
+
module Contrib
|
|
6
|
+
module Rack
|
|
7
|
+
module ResponseBody
|
|
8
|
+
# NOTE: We compute content length only for fixed-size response bodies,
|
|
9
|
+
# ignoring streaming bodies to avoid buffering.
|
|
10
|
+
#
|
|
11
|
+
# On Rack 3.x, `body.to_ary` on a BodyProxy triggers `close` on all
|
|
12
|
+
# nested BodyProxy layers. This is safe because web servers, like Puma
|
|
13
|
+
# handles already-closed bodies (its own `to_ary` becomes a no-op).
|
|
14
|
+
#
|
|
15
|
+
# @see Puma::Response#prepare_response
|
|
16
|
+
# @see https://github.com/puma/puma/blob/b1271222cbf21868f3fb64154caa4d03936a7b9e/lib/puma/response.rb#L165-L168
|
|
17
|
+
def self.content_length(body)
|
|
18
|
+
return unless body.respond_to?(:to_ary)
|
|
19
|
+
|
|
20
|
+
# NOTE: When `to_ary` exists but returns `nil`, the body will be
|
|
21
|
+
# transferred in chunks and we can't compute content length
|
|
22
|
+
# without buffering it.
|
|
23
|
+
body_ary = body.to_ary
|
|
24
|
+
return unless body_ary.is_a?(Array)
|
|
25
|
+
|
|
26
|
+
body_ary.sum { |part| part.is_a?(String) ? part.bytesize : 0 }
|
|
27
|
+
rescue => e
|
|
28
|
+
AppSec.telemetry.report(e, description: 'AppSec: Failed to compute body content length')
|
|
29
|
+
|
|
30
|
+
nil
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -21,7 +21,7 @@ module Datadog
|
|
|
21
21
|
end
|
|
22
22
|
|
|
23
23
|
def watch_request_action(gateway = Instrumentation.gateway)
|
|
24
|
-
gateway.watch('rails.request.action'
|
|
24
|
+
gateway.watch('rails.request.action') do |stack, gateway_request|
|
|
25
25
|
context = gateway_request.env[AppSec::Ext::CONTEXT_KEY]
|
|
26
26
|
|
|
27
27
|
persistent_data = {
|
|
@@ -47,7 +47,7 @@ module Datadog
|
|
|
47
47
|
end
|
|
48
48
|
|
|
49
49
|
def watch_response_body_json(gateway = Instrumentation.gateway)
|
|
50
|
-
gateway.watch('rails.response.body.json'
|
|
50
|
+
gateway.watch('rails.response.body.json') do |stack, container|
|
|
51
51
|
context = container.context
|
|
52
52
|
|
|
53
53
|
persistent_data = {
|