datadog 2.36.0 → 2.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -1
- data/ext/datadog_profiling_native_extension/clock_id_from_mach.c +5 -12
- data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +6 -3
- data/ext/datadog_profiling_native_extension/collectors_idle_sampling_helper.c +17 -3
- data/ext/datadog_profiling_native_extension/collectors_thread_context.c +219 -98
- data/ext/datadog_profiling_native_extension/collectors_thread_context.h +2 -0
- data/ext/datadog_profiling_native_extension/extconf.rb +11 -0
- data/ext/datadog_profiling_native_extension/ruby_helpers.c +7 -0
- data/ext/libdatadog_api/di.c +76 -0
- data/ext/libdatadog_api/extconf.rb +6 -0
- data/ext/libdatadog_extconf_helpers.rb +1 -1
- data/lib/datadog/ai_guard/api_client.rb +6 -4
- data/lib/datadog/appsec/api_security/route_extractor.rb +7 -4
- data/lib/datadog/appsec/configuration.rb +6 -0
- data/lib/datadog/appsec/context.rb +4 -2
- data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb +59 -4
- data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb +59 -5
- data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb +1 -1
- data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +4 -0
- data/lib/datadog/appsec/contrib/rack/input_peeker.rb +4 -17
- data/lib/datadog/appsec/contrib/rack/request_middleware.rb +21 -0
- data/lib/datadog/appsec/contrib/rails/gateway/request.rb +2 -1
- data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb +60 -7
- data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +4 -0
- data/lib/datadog/appsec/ext.rb +1 -0
- data/lib/datadog/appsec/metrics/collector.rb +18 -1
- data/lib/datadog/appsec/metrics/exporter.rb +9 -3
- data/lib/datadog/appsec/remote.rb +9 -3
- data/lib/datadog/appsec/route_normalizer/rails_route_pattern.rb +176 -0
- data/lib/datadog/appsec/route_normalizer/route_pattern.rb +378 -0
- data/lib/datadog/appsec/route_normalizer/route_text.rb +57 -0
- data/lib/datadog/appsec/route_normalizer.rb +80 -0
- data/lib/datadog/appsec/utils/http/body_reader.rb +61 -0
- data/lib/datadog/core/configuration/components.rb +82 -2
- data/lib/datadog/core/configuration/components_state.rb +6 -1
- data/lib/datadog/core/configuration/settings.rb +0 -3
- data/lib/datadog/core/configuration/supported_configurations.rb +2 -0
- data/lib/datadog/core/evp.rb +11 -0
- data/lib/datadog/core/remote/client/capabilities.rb +34 -7
- data/lib/datadog/core/telemetry/event/app_started.rb +8 -1
- data/lib/datadog/di/base.rb +3 -1
- data/lib/datadog/di/code_tracker.rb +5 -2
- data/lib/datadog/di/component.rb +110 -36
- data/lib/datadog/di/error.rb +10 -0
- data/lib/datadog/di/fatal_exceptions.rb +26 -0
- data/lib/datadog/di/instrumenter.rb +381 -165
- data/lib/datadog/di/probe.rb +14 -0
- data/lib/datadog/di/probe_file_loader.rb +8 -3
- data/lib/datadog/di/probe_manager.rb +67 -9
- data/lib/datadog/di/probe_notification_builder.rb +4 -1
- data/lib/datadog/di/probe_notifier_worker.rb +52 -32
- data/lib/datadog/di/redactor.rb +16 -1
- data/lib/datadog/di/remote.rb +175 -14
- data/lib/datadog/di/serializer.rb +12 -5
- data/lib/datadog/di/transport/input.rb +8 -4
- data/lib/datadog/di.rb +63 -0
- data/lib/datadog/kit/appsec/events/v2.rb +60 -2
- data/lib/datadog/open_feature/component.rb +29 -6
- data/lib/datadog/open_feature/configuration.rb +8 -0
- data/lib/datadog/open_feature/ext.rb +2 -0
- data/lib/datadog/open_feature/flag_evaluation/aggregator.rb +286 -0
- data/lib/datadog/open_feature/flag_evaluation/writer.rb +433 -0
- data/lib/datadog/open_feature/hooks/flag_eval_evp_hook.rb +101 -0
- data/lib/datadog/open_feature/hooks/{flag_eval_hook.rb → flag_eval_metrics_hook.rb} +1 -1
- data/lib/datadog/open_feature/native_evaluator.rb +22 -0
- data/lib/datadog/open_feature/provider.rb +59 -27
- data/lib/datadog/open_feature/transport.rb +55 -1
- data/lib/datadog/opentelemetry/sdk/logs_exporter.rb +5 -12
- data/lib/datadog/opentelemetry/sdk/metrics_exporter.rb +4 -11
- data/lib/datadog/opentelemetry/sdk.rb +19 -0
- data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +1 -1
- data/lib/datadog/profiling/collectors/idle_sampling_helper.rb +5 -3
- data/lib/datadog/symbol_database/component.rb +164 -47
- data/lib/datadog/symbol_database/configuration.rb +12 -4
- data/lib/datadog/symbol_database/extensions.rb +19 -0
- data/lib/datadog/symbol_database/extractor.rb +124 -87
- data/lib/datadog/symbol_database/file_hash.rb +3 -1
- data/lib/datadog/symbol_database/remote.rb +8 -3
- data/lib/datadog/symbol_database/scope_batcher.rb +7 -3
- data/lib/datadog/symbol_database/uploader.rb +9 -2
- data/lib/datadog/symbol_database.rb +22 -0
- data/lib/datadog/tracing/distributed/baggage.rb +2 -1
- data/lib/datadog/tracing/ext.rb +9 -0
- data/lib/datadog/tracing/remote.rb +34 -3
- data/lib/datadog/version.rb +1 -1
- data/lib/datadog.rb +1 -0
- metadata +19 -6
|
@@ -21,6 +21,8 @@ VALUE enforce_thread_context_collector_instance(VALUE object);
|
|
|
21
21
|
void thread_context_collector_stats(VALUE self_instance, VALUE stats_hash);
|
|
22
22
|
void thread_context_collector_stats_reset_not_thread_safe(VALUE self_instance);
|
|
23
23
|
void thread_context_collector_on_serialize(VALUE self_instance);
|
|
24
|
+
void thread_context_collector_profiler_internal_thread_started(void);
|
|
25
|
+
void thread_context_collector_profiler_internal_thread_done(VALUE self_instance);
|
|
24
26
|
|
|
25
27
|
#ifndef NO_GVL_INSTRUMENTATION
|
|
26
28
|
typedef enum {
|
|
@@ -83,6 +83,12 @@ Logging.message("[datadog] End of compiler information\n")
|
|
|
83
83
|
# But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
|
|
84
84
|
append_cflags "-Werror" if ENV["DATADOG_GEM_CI"] == "true"
|
|
85
85
|
|
|
86
|
+
# TEMPORARY STOPGAP: libdatadog v36's vendored `common.h` ships duplicate typedefs
|
|
87
|
+
# (a header-dedup regression), which `-Werror` turns into fatal
|
|
88
|
+
# `-Wtypedef-redefinition` errors under C11/gnu11. Keep it a warning until a fixed
|
|
89
|
+
# header lands upstream in libdatadog/libdatadog-rb. Remove this once that ships.
|
|
90
|
+
append_cflags "-Wno-error=typedef-redefinition" if ENV["DATADOG_GEM_CI"] == "true"
|
|
91
|
+
|
|
86
92
|
# Older gcc releases may not default to C99 and we need to ask for this. This is also used:
|
|
87
93
|
# * by upstream Ruby -- search for gnu99 in the codebase
|
|
88
94
|
# * by msgpack, another datadog gem dependency
|
|
@@ -154,6 +160,11 @@ $defs << "-DNO_IMEMO_OBJECT_ID" unless RUBY_VERSION < "4"
|
|
|
154
160
|
# (see https://bugs.ruby-lang.org/issues/21710)
|
|
155
161
|
$defs << "-DUSE_DEFERRED_HEAP_ALLOCATION_RECORDING" unless RUBY_VERSION < "4"
|
|
156
162
|
|
|
163
|
+
# On Ruby 4.0, we've seen crashes when computing the memsize of a class/module/iclass:
|
|
164
|
+
# rb_obj_memsize_of walks the per-namespace class extensions (classext_memsize), which seem to sometimes be in an inconsistent state
|
|
165
|
+
# (see https://github.com/DataDog/dd-trace-rb/issues/5936)
|
|
166
|
+
$defs << "-DNO_SAFE_CLASS_MEMSIZE" unless RUBY_VERSION < "4"
|
|
167
|
+
|
|
157
168
|
# This symbol is exclusively visible on certain Ruby versions: 2.6 to 3.2, as well as 3.4 (but not 4.0+)
|
|
158
169
|
# It's only used to get extra information about an object when a failure happens, so it's a "very nice to have" but not
|
|
159
170
|
# actually required for correct behavior of the profiler.
|
|
@@ -162,9 +162,16 @@ size_t rb_obj_memsize_of(VALUE obj);
|
|
|
162
162
|
size_t ruby_obj_memsize_of(VALUE obj) {
|
|
163
163
|
switch (rb_type(obj)) {
|
|
164
164
|
case T_OBJECT:
|
|
165
|
+
// On Ruby 4.0, computing the size of a class/module/iclass seems to not be safe: `rb_obj_memsize_of` walks the
|
|
166
|
+
// per-namespace class extensions (`rb_class_classext_foreach` -> `classext_memsize` -> `rb_id_table_memsize`)
|
|
167
|
+
// and can crash the VM when called on objects tracked by heap profiling.
|
|
168
|
+
// See https://github.com/DataDog/dd-trace-rb/issues/5936. Class objects contribute negligibly to heap size,
|
|
169
|
+
// so we skip them (fall through to the `default` branch, returning 0) rather than risk a SIGSEGV.
|
|
170
|
+
#ifndef NO_SAFE_CLASS_MEMSIZE
|
|
165
171
|
case T_MODULE:
|
|
166
172
|
case T_CLASS:
|
|
167
173
|
case T_ICLASS:
|
|
174
|
+
#endif
|
|
168
175
|
case T_STRING:
|
|
169
176
|
case T_ARRAY:
|
|
170
177
|
case T_HASH:
|
data/ext/libdatadog_api/di.c
CHANGED
|
@@ -21,6 +21,12 @@ void rb_objspace_each_objects(
|
|
|
21
21
|
// from standard library exception classes like NameError.
|
|
22
22
|
static ID id_mesg;
|
|
23
23
|
|
|
24
|
+
// ID for the fiber-local key that backs the method-probe re-entrancy guard.
|
|
25
|
+
// Storage is the same hashtable that backs Thread#[] / Thread#[]=, but accessed
|
|
26
|
+
// directly via rb_thread_local_aref / rb_thread_local_aset so that user-installed
|
|
27
|
+
// method probes on Thread#[] / Thread#[]= cannot intercept guard reads/writes.
|
|
28
|
+
static ID id_datadog_di_in_probe;
|
|
29
|
+
|
|
24
30
|
// Returns whether the argument is an IMEMO of type ISEQ.
|
|
25
31
|
static bool ddtrace_imemo_iseq_p(VALUE v) {
|
|
26
32
|
return rb_objspace_internal_object_p(v) && RB_TYPE_P(v, T_IMEMO) && ddtrace_imemo_type(v) == IMEMO_TYPE_ISEQ;
|
|
@@ -75,6 +81,71 @@ static VALUE exception_message(DDTRACE_UNUSED VALUE _self, VALUE exception) {
|
|
|
75
81
|
return rb_ivar_get(exception, id_mesg);
|
|
76
82
|
}
|
|
77
83
|
|
|
84
|
+
/*
|
|
85
|
+
* call-seq:
|
|
86
|
+
* DI.in_probe? -> true | false
|
|
87
|
+
*
|
|
88
|
+
* Returns whether the current fiber is currently inside DI probe processing.
|
|
89
|
+
* Reads the same fiber-local storage as Thread.current[:datadog_di_in_probe]
|
|
90
|
+
* but bypasses Thread#[] method dispatch — a user method probe on Thread#[]
|
|
91
|
+
* cannot observe or intercept this call.
|
|
92
|
+
*
|
|
93
|
+
* @api private
|
|
94
|
+
*/
|
|
95
|
+
static VALUE in_probe_p(DDTRACE_UNUSED VALUE _self) {
|
|
96
|
+
VALUE v = rb_thread_local_aref(rb_thread_current(), id_datadog_di_in_probe);
|
|
97
|
+
return RTEST(v) ? Qtrue : Qfalse;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/*
|
|
101
|
+
* call-seq:
|
|
102
|
+
* DI.enter_probe -> nil
|
|
103
|
+
*
|
|
104
|
+
* Marks the current fiber as inside DI probe processing. Writes to the same
|
|
105
|
+
* fiber-local storage as Thread.current[:datadog_di_in_probe] = true, but
|
|
106
|
+
* bypasses Thread#[]= method dispatch — a user method probe on Thread#[]=
|
|
107
|
+
* cannot observe or intercept this call.
|
|
108
|
+
*
|
|
109
|
+
* @api private
|
|
110
|
+
*/
|
|
111
|
+
static VALUE enter_probe(DDTRACE_UNUSED VALUE _self) {
|
|
112
|
+
rb_thread_local_aset(rb_thread_current(), id_datadog_di_in_probe, Qtrue);
|
|
113
|
+
return Qnil;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/*
|
|
117
|
+
* call-seq:
|
|
118
|
+
* DI.leave_probe -> nil
|
|
119
|
+
*
|
|
120
|
+
* Marks the current fiber as no longer inside DI probe processing. Writes to
|
|
121
|
+
* the same fiber-local storage as Thread.current[:datadog_di_in_probe] = nil,
|
|
122
|
+
* but bypasses Thread#[]= method dispatch — a user method probe on Thread#[]=
|
|
123
|
+
* cannot observe or intercept this call.
|
|
124
|
+
*
|
|
125
|
+
* @api private
|
|
126
|
+
*/
|
|
127
|
+
static VALUE leave_probe(DDTRACE_UNUSED VALUE _self) {
|
|
128
|
+
rb_thread_local_aset(rb_thread_current(), id_datadog_di_in_probe, Qnil);
|
|
129
|
+
return Qnil;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/*
|
|
133
|
+
* call-seq:
|
|
134
|
+
* DI.hash?(obj) -> true | false
|
|
135
|
+
*
|
|
136
|
+
* Returns whether the given object is a Hash via a direct type check
|
|
137
|
+
* (RB_TYPE_P), bypassing Kernel#is_a? method dispatch. Used in the method
|
|
138
|
+
* probe wrapper (the Ruby < 3 argument-serialization split) to test whether
|
|
139
|
+
* the trailing argument is a hash without giving a user-installed method
|
|
140
|
+
* probe on Kernel#is_a? a chance to recurse. Matches Hash and its
|
|
141
|
+
* subclasses, like is_a?(Hash), since their instances are still T_HASH.
|
|
142
|
+
*
|
|
143
|
+
* @api private
|
|
144
|
+
*/
|
|
145
|
+
static VALUE is_hash(DDTRACE_UNUSED VALUE _self, VALUE obj) {
|
|
146
|
+
return RB_TYPE_P(obj, T_HASH) ? Qtrue : Qfalse;
|
|
147
|
+
}
|
|
148
|
+
|
|
78
149
|
// rb_iseq_type was added in Ruby 3.1 (commit 89a02d89 by Koichi Sasada,
|
|
79
150
|
// 2021-12-19). It returns the iseq type as a Symbol. On Ruby < 3.1 this
|
|
80
151
|
// function does not exist, so have_func('rb_iseq_type') in extconf.rb
|
|
@@ -117,10 +188,15 @@ static VALUE iseq_type(DDTRACE_UNUSED VALUE _self, VALUE iseq_val) {
|
|
|
117
188
|
|
|
118
189
|
void di_init(VALUE datadog_module) {
|
|
119
190
|
id_mesg = rb_intern("mesg");
|
|
191
|
+
id_datadog_di_in_probe = rb_intern("datadog_di_in_probe");
|
|
120
192
|
|
|
121
193
|
VALUE di_module = rb_define_module_under(datadog_module, "DI");
|
|
122
194
|
rb_define_singleton_method(di_module, "all_iseqs", all_iseqs, 0);
|
|
123
195
|
rb_define_singleton_method(di_module, "exception_message", exception_message, 1);
|
|
196
|
+
rb_define_singleton_method(di_module, "in_probe?", in_probe_p, 0);
|
|
197
|
+
rb_define_singleton_method(di_module, "enter_probe", enter_probe, 0);
|
|
198
|
+
rb_define_singleton_method(di_module, "leave_probe", leave_probe, 0);
|
|
199
|
+
rb_define_singleton_method(di_module, "hash?", is_hash, 1);
|
|
124
200
|
#ifdef HAVE_RB_ISEQ_TYPE
|
|
125
201
|
rb_define_singleton_method(di_module, "iseq_type", iseq_type, 1);
|
|
126
202
|
#endif
|
|
@@ -41,6 +41,12 @@ Datadog::LibdatadogExtconfHelpers.dump_mkmf_log_on_failure!
|
|
|
41
41
|
# But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
|
|
42
42
|
append_cflags '-Werror' if ENV['DATADOG_GEM_CI'] == 'true'
|
|
43
43
|
|
|
44
|
+
# TEMPORARY STOPGAP: libdatadog v36's vendored `common.h` ships duplicate typedefs
|
|
45
|
+
# (a header-dedup regression), which `-Werror` turns into fatal
|
|
46
|
+
# `-Wtypedef-redefinition` errors under C11/gnu11. Keep it a warning until a fixed
|
|
47
|
+
# header lands upstream in libdatadog/libdatadog-rb. Remove this once that ships.
|
|
48
|
+
append_cflags '-Wno-error=typedef-redefinition' if ENV['DATADOG_GEM_CI'] == 'true'
|
|
49
|
+
|
|
44
50
|
# Older gcc releases may not default to C99 and we need to ask for this. This is also used:
|
|
45
51
|
# * by upstream Ruby -- search for gnu99 in the codebase
|
|
46
52
|
# * by msgpack, another datadog gem dependency
|
|
@@ -10,7 +10,7 @@ module Datadog
|
|
|
10
10
|
module LibdatadogExtconfHelpers
|
|
11
11
|
# Used to make sure the correct gem version gets loaded, as extconf.rb does not get run with "bundle exec" and thus
|
|
12
12
|
# may see multiple libdatadog versions. See https://github.com/DataDog/dd-trace-rb/pull/2531 for the horror story.
|
|
13
|
-
LIBDATADOG_VERSION = '~>
|
|
13
|
+
LIBDATADOG_VERSION = '~> 36.0.0.1.0'
|
|
14
14
|
|
|
15
15
|
# Used as an workaround for a limitation with how dynamic linking works in environments where the datadog gem and
|
|
16
16
|
# libdatadog are moved after the extension gets compiled.
|
|
@@ -17,11 +17,13 @@ module Datadog
|
|
|
17
17
|
|
|
18
18
|
@endpoint_uri = if endpoint
|
|
19
19
|
URI(endpoint) #: URI::HTTP
|
|
20
|
+
elsif Datadog.configuration.site
|
|
21
|
+
host = Datadog.configuration.site.dup
|
|
22
|
+
host.prepend("app.") if host.count(".") == 1
|
|
23
|
+
|
|
24
|
+
URI::HTTPS.build(host: host, path: DEFAULT_PATH)
|
|
20
25
|
else
|
|
21
|
-
URI::HTTPS.build(
|
|
22
|
-
host: Datadog.configuration.site || DEFAULT_SITE,
|
|
23
|
-
path: DEFAULT_PATH
|
|
24
|
-
)
|
|
26
|
+
URI::HTTPS.build(host: DEFAULT_SITE, path: DEFAULT_PATH)
|
|
25
27
|
end
|
|
26
28
|
|
|
27
29
|
@headers = {
|
|
@@ -31,16 +31,19 @@ module Datadog
|
|
|
31
31
|
# uses `sinatra.route` with a string like "GET /users/:id"
|
|
32
32
|
# Grape
|
|
33
33
|
# uses `grape.routing_args` with a hash with a `:route_info` key
|
|
34
|
-
# that contains a
|
|
35
|
-
#
|
|
34
|
+
# that contains a {Grape::Router::Route} object that contains
|
|
35
|
+
# {Grape::Router::Pattern} object with an `origin` method
|
|
36
|
+
# Rails with `action_pack` tracing contrib (fast path)
|
|
37
|
+
# `datadog.action_dispatch.route` stores the {Journey::Route}
|
|
38
|
+
# object set by the tracer at routing time
|
|
36
39
|
# Rails < 7.1 (slow path)
|
|
37
|
-
# uses `action_dispatch.routes` to store
|
|
40
|
+
# uses `action_dispatch.routes` to store {ActionDispatch::Routing::RouteSet}
|
|
38
41
|
# which can recognize requests
|
|
39
42
|
# Rails > 7.1 (fast path)
|
|
40
43
|
# uses `action_dispatch.route_uri_pattern` with a string like
|
|
41
44
|
# "/users/:id(.:format)"
|
|
42
45
|
# Rails > 8.1.1 (fast path)
|
|
43
|
-
# uses `action_dispatch.route` to store the ActionDispatch::Journey::Route
|
|
46
|
+
# uses `action_dispatch.route` to store the {ActionDispatch::Journey::Route}
|
|
44
47
|
# that matched when the request was routed
|
|
45
48
|
#
|
|
46
49
|
# WARNING: This method works only *after* the request has been routed.
|
|
@@ -414,6 +414,12 @@ module Datadog
|
|
|
414
414
|
o.env 'DD_API_SECURITY_MAX_DOWNSTREAM_REQUEST_BODY_ANALYSIS'
|
|
415
415
|
o.default 1
|
|
416
416
|
end
|
|
417
|
+
|
|
418
|
+
option :max_downstream_body_bytes do |o|
|
|
419
|
+
o.type :int
|
|
420
|
+
o.env 'DD_API_SECURITY_MAX_DOWNSTREAM_BODY_BYTES'
|
|
421
|
+
o.default 10_485_760
|
|
422
|
+
end
|
|
417
423
|
end
|
|
418
424
|
end
|
|
419
425
|
|
|
@@ -20,7 +20,7 @@ module Datadog
|
|
|
20
20
|
ActiveContextError = Class.new(StandardError) # steep:ignore IncompatibleAssignment
|
|
21
21
|
|
|
22
22
|
# TODO: add delegators for active trace span
|
|
23
|
-
attr_reader :trace, :span
|
|
23
|
+
attr_reader :trace, :span, :metrics
|
|
24
24
|
|
|
25
25
|
# Shared mutable storage for counters, flags, and data accumulated during
|
|
26
26
|
# the request's lifecycle.
|
|
@@ -103,7 +103,8 @@ module Datadog
|
|
|
103
103
|
end
|
|
104
104
|
|
|
105
105
|
def extract_schema!
|
|
106
|
-
|
|
106
|
+
persistent_data = {'waf.context.processor' => {'extract-schema' => true}}
|
|
107
|
+
waf_result = run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
|
|
107
108
|
security_event = AppSec::SecurityEvent.new(waf_result, trace: trace, span: span)
|
|
108
109
|
|
|
109
110
|
@state[:schema_extracted] = security_event.schema?
|
|
@@ -116,6 +117,7 @@ module Datadog
|
|
|
116
117
|
|
|
117
118
|
Metrics::Exporter.export_waf_metrics(@metrics.waf, @span)
|
|
118
119
|
Metrics::Exporter.export_rasp_metrics(@metrics.rasp, @span)
|
|
120
|
+
Metrics::Exporter.export_downstream_response_metrics(@metrics.downstream_responses, @span)
|
|
119
121
|
end
|
|
120
122
|
|
|
121
123
|
def export_request_telemetry
|
|
@@ -7,6 +7,8 @@ require_relative '../../trace_keeper'
|
|
|
7
7
|
require_relative '../../security_event'
|
|
8
8
|
require_relative '../../utils/http/url_encoded'
|
|
9
9
|
require_relative '../../utils/http/body'
|
|
10
|
+
require_relative '../../utils/http/body_reader'
|
|
11
|
+
require_relative '../../utils/http/media_type'
|
|
10
12
|
|
|
11
13
|
module Datadog
|
|
12
14
|
module AppSec
|
|
@@ -39,7 +41,7 @@ module Datadog
|
|
|
39
41
|
end
|
|
40
42
|
|
|
41
43
|
if !is_redirect && data[SAMPLE_BODY_KEY]
|
|
42
|
-
body =
|
|
44
|
+
body = parse_request_body(data[:body], content_type: headers['content-type'])
|
|
43
45
|
ephemeral_data['server.io.net.request.body'] = body if body
|
|
44
46
|
end
|
|
45
47
|
|
|
@@ -67,7 +69,7 @@ module Datadog
|
|
|
67
69
|
end
|
|
68
70
|
|
|
69
71
|
if !is_redirect && data[SAMPLE_BODY_KEY]
|
|
70
|
-
body =
|
|
72
|
+
body = parse_response_body(data.dig(:response, :body), headers: headers, context: context)
|
|
71
73
|
ephemeral_data['server.io.net.response.body'] = body if body
|
|
72
74
|
end
|
|
73
75
|
|
|
@@ -89,11 +91,64 @@ module Datadog
|
|
|
89
91
|
data[SAMPLE_BODY_KEY] = true
|
|
90
92
|
end
|
|
91
93
|
|
|
92
|
-
def
|
|
94
|
+
def parse_request_body(body, content_type:)
|
|
93
95
|
media_type = Utils::HTTP::MediaType.parse(content_type)
|
|
94
96
|
return unless media_type
|
|
95
97
|
|
|
96
|
-
|
|
98
|
+
limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
|
|
99
|
+
content = read_body(body, limit: limit)
|
|
100
|
+
return if content.nil? || content.bytesize > limit
|
|
101
|
+
|
|
102
|
+
Utils::HTTP::Body.parse(content, media_type: media_type)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def parse_response_body(body, headers:, context:)
|
|
106
|
+
return unless readable_body?(body)
|
|
107
|
+
|
|
108
|
+
media_type = Utils::HTTP::MediaType.parse(headers['content-type'])
|
|
109
|
+
if !media_type || media_type.type != 'application'
|
|
110
|
+
context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
|
|
111
|
+
return
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
subtype = media_type.subtype
|
|
115
|
+
if subtype != 'json' && !subtype.end_with?('+json') && subtype != 'x-www-form-urlencoded'
|
|
116
|
+
context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
|
|
117
|
+
return
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
content_length_value = headers['content-length']
|
|
121
|
+
if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
|
|
122
|
+
context.metrics.record_ignored_downstream_response_body(:content_length_missing)
|
|
123
|
+
return
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
content_length = content_length_value.to_i
|
|
127
|
+
max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
|
|
128
|
+
if content_length > max
|
|
129
|
+
context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
|
|
130
|
+
return
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
content = read_body(body, limit: content_length)
|
|
134
|
+
return if content.nil?
|
|
135
|
+
|
|
136
|
+
if content.bytesize > content_length
|
|
137
|
+
context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
|
|
138
|
+
return
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
Utils::HTTP::Body.parse(content, media_type: media_type)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def readable_body?(body)
|
|
145
|
+
body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def read_body(body, limit:)
|
|
149
|
+
Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
|
|
150
|
+
rescue
|
|
151
|
+
nil
|
|
97
152
|
end
|
|
98
153
|
|
|
99
154
|
def request_url(data)
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
require_relative '../../event'
|
|
4
4
|
require_relative '../../trace_keeper'
|
|
5
5
|
require_relative '../../security_event'
|
|
6
|
-
require_relative '../../utils/http/media_type'
|
|
7
6
|
require_relative '../../utils/http/body'
|
|
7
|
+
require_relative '../../utils/http/body_reader'
|
|
8
|
+
require_relative '../../utils/http/media_type'
|
|
8
9
|
|
|
9
10
|
module Datadog
|
|
10
11
|
module AppSec
|
|
@@ -37,7 +38,7 @@ module Datadog
|
|
|
37
38
|
end
|
|
38
39
|
|
|
39
40
|
if !is_redirect && env[SAMPLE_BODY_KEY]
|
|
40
|
-
body =
|
|
41
|
+
body = parse_request_body(env.body, content_type: headers['content-type'])
|
|
41
42
|
ephemeral_data['server.io.net.request.body'] = body if body
|
|
42
43
|
end
|
|
43
44
|
|
|
@@ -64,7 +65,7 @@ module Datadog
|
|
|
64
65
|
end
|
|
65
66
|
|
|
66
67
|
if !is_redirect && env[SAMPLE_BODY_KEY]
|
|
67
|
-
body =
|
|
68
|
+
body = parse_response_body(env.body, headers: headers, context: context)
|
|
68
69
|
ephemeral_data['server.io.net.response.body'] = body if body
|
|
69
70
|
end
|
|
70
71
|
|
|
@@ -82,11 +83,64 @@ module Datadog
|
|
|
82
83
|
env[SAMPLE_BODY_KEY] = true
|
|
83
84
|
end
|
|
84
85
|
|
|
85
|
-
def
|
|
86
|
+
def parse_request_body(body, content_type:)
|
|
86
87
|
media_type = Utils::HTTP::MediaType.parse(content_type)
|
|
87
88
|
return unless media_type
|
|
88
89
|
|
|
89
|
-
|
|
90
|
+
limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
|
|
91
|
+
content = read_body(body, limit: limit)
|
|
92
|
+
return if content.nil? || content.bytesize > limit
|
|
93
|
+
|
|
94
|
+
Utils::HTTP::Body.parse(content, media_type: media_type)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def parse_response_body(body, headers:, context:)
|
|
98
|
+
return unless readable_body?(body)
|
|
99
|
+
|
|
100
|
+
media_type = Utils::HTTP::MediaType.parse(headers['content-type'])
|
|
101
|
+
if !media_type || media_type.type != 'application'
|
|
102
|
+
context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
|
|
103
|
+
return
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
subtype = media_type.subtype
|
|
107
|
+
if subtype != 'json' && !subtype.end_with?('+json') && subtype != 'x-www-form-urlencoded'
|
|
108
|
+
context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
|
|
109
|
+
return
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
content_length_value = headers['content-length']
|
|
113
|
+
if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
|
|
114
|
+
context.metrics.record_ignored_downstream_response_body(:content_length_missing)
|
|
115
|
+
return
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
content_length = content_length_value.to_i
|
|
119
|
+
max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
|
|
120
|
+
if content_length > max
|
|
121
|
+
context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
|
|
122
|
+
return
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
content = read_body(body, limit: content_length)
|
|
126
|
+
return if content.nil?
|
|
127
|
+
|
|
128
|
+
if content.bytesize > content_length
|
|
129
|
+
context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
|
|
130
|
+
return
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
Utils::HTTP::Body.parse(content, media_type: media_type)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def readable_body?(body)
|
|
137
|
+
body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def read_body(body, limit:)
|
|
141
|
+
Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
|
|
142
|
+
rescue
|
|
143
|
+
nil
|
|
90
144
|
end
|
|
91
145
|
|
|
92
146
|
def normalize_headers(headers)
|
|
@@ -72,11 +72,11 @@ module Datadog
|
|
|
72
72
|
args_hash, fragment_name, nil, arguments_from_directives(fragment.directives, query_variables)
|
|
73
73
|
)
|
|
74
74
|
|
|
75
|
+
# re-used fragments are not expanded
|
|
75
76
|
visited_fragments[fragment_name] = true
|
|
76
77
|
arguments_from_selections(
|
|
77
78
|
fragment.selections, query_variables, args_hash, fragments, visited_fragments
|
|
78
79
|
)
|
|
79
|
-
visited_fragments.delete(fragment_name)
|
|
80
80
|
when ::GraphQL::Language::Nodes::Field
|
|
81
81
|
selection_name = selection.alias || selection.name
|
|
82
82
|
field_arguments = arguments_hash(selection.arguments, query_variables) unless selection.arguments.empty?
|
|
@@ -104,6 +104,10 @@ module Datadog
|
|
|
104
104
|
body = gateway_request.form_hash
|
|
105
105
|
persistent_data['server.request.body'] = body if body
|
|
106
106
|
end
|
|
107
|
+
# NOTE: Body was parsed before measurement, keep byte_length unset
|
|
108
|
+
elsif gateway_request.env.key?('rack.request.form_hash')
|
|
109
|
+
body = gateway_request.env['rack.request.form_hash']
|
|
110
|
+
persistent_data['server.request.body'] = body if body
|
|
107
111
|
end
|
|
108
112
|
|
|
109
113
|
next stack.call(request) if persistent_data.empty?
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require 'stringio'
|
|
4
4
|
require_relative 'buffered_input'
|
|
5
|
+
require_relative '../../utils/http/body_reader'
|
|
5
6
|
|
|
6
7
|
module Datadog
|
|
7
8
|
module AppSec
|
|
@@ -38,7 +39,9 @@ module Datadog
|
|
|
38
39
|
# in case an upstream framework already consumed part of the stream
|
|
39
40
|
return if rewindable && !rewind(rack_input)
|
|
40
41
|
|
|
41
|
-
|
|
42
|
+
# NOTE: Read one byte past the limit to distinguish an exact-limit body
|
|
43
|
+
# from an over-limit body without reading the whole stream.
|
|
44
|
+
buffer = Utils::HTTP::BodyReader.read_stream(rack_input, limit: limit)
|
|
42
45
|
over_limit = buffer.bytesize > limit
|
|
43
46
|
|
|
44
47
|
if rewindable
|
|
@@ -69,22 +72,6 @@ module Datadog
|
|
|
69
72
|
Datadog.logger.debug { "AppSec: Failed to rewind `rack.input`: #{e.class}: #{e.message}" }
|
|
70
73
|
false
|
|
71
74
|
end
|
|
72
|
-
|
|
73
|
-
private_class_method def peek(io, limit)
|
|
74
|
-
# NOTE: Read one byte past the limit to distinguish an exact-limit body
|
|
75
|
-
# from an over-limit body without reading the whole stream.
|
|
76
|
-
max = limit + 1
|
|
77
|
-
buffer = +''.b
|
|
78
|
-
|
|
79
|
-
while buffer.bytesize <= limit
|
|
80
|
-
chunk = io.read(max - buffer.bytesize)
|
|
81
|
-
break if chunk.nil? || chunk.empty?
|
|
82
|
-
|
|
83
|
-
buffer << chunk
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
buffer
|
|
87
|
-
end
|
|
88
75
|
end
|
|
89
76
|
end
|
|
90
77
|
end
|
|
@@ -10,6 +10,7 @@ require_relative '../../event'
|
|
|
10
10
|
require_relative '../../response'
|
|
11
11
|
require_relative '../../api_security'
|
|
12
12
|
require_relative '../../default_header_tags'
|
|
13
|
+
require_relative '../../route_normalizer'
|
|
13
14
|
require_relative '../../security_event'
|
|
14
15
|
require_relative '../../instrumentation/gateway'
|
|
15
16
|
|
|
@@ -76,6 +77,8 @@ module Datadog
|
|
|
76
77
|
nil
|
|
77
78
|
end
|
|
78
79
|
|
|
80
|
+
add_normalized_route_tag(ctx, env)
|
|
81
|
+
|
|
79
82
|
if interrupt_params
|
|
80
83
|
ctx.mark_as_interrupted!
|
|
81
84
|
http_response = AppSec::Response.from_interrupt_params(interrupt_params, env['HTTP_ACCEPT']).to_rack
|
|
@@ -198,6 +201,24 @@ module Datadog
|
|
|
198
201
|
end
|
|
199
202
|
end
|
|
200
203
|
|
|
204
|
+
def add_normalized_route_tag(context, env)
|
|
205
|
+
return unless AppSec::APISecurity.enabled?
|
|
206
|
+
|
|
207
|
+
span = context.span
|
|
208
|
+
return unless span
|
|
209
|
+
|
|
210
|
+
pattern = context.trace&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE)
|
|
211
|
+
return unless pattern
|
|
212
|
+
|
|
213
|
+
# NOTE: To build full path that covers mounted engines we need to add
|
|
214
|
+
# pre-computed by Tracer route path tag to the normalized route
|
|
215
|
+
prefix = context.trace&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH) || env['SCRIPT_NAME']
|
|
216
|
+
normalized_route = RouteNormalizer.extract_normalized_route(env, prefix: prefix, pattern: pattern)
|
|
217
|
+
return unless normalized_route
|
|
218
|
+
|
|
219
|
+
span.set_tag(AppSec::Ext::TAG_NORMALIZED_ROUTE, "#{prefix}#{normalized_route}")
|
|
220
|
+
end
|
|
221
|
+
|
|
201
222
|
def oneshot_tags_sent?
|
|
202
223
|
@oneshot_tags_sent
|
|
203
224
|
end
|
|
@@ -47,6 +47,7 @@ module Datadog
|
|
|
47
47
|
body = request.env['action_dispatch.request.request_parameters']
|
|
48
48
|
|
|
49
49
|
return if body.nil?
|
|
50
|
+
return body unless request.env['action_dispatch.request.path_parameters']
|
|
50
51
|
|
|
51
52
|
body.reject do |k, _v|
|
|
52
53
|
request.env['action_dispatch.request.path_parameters'].key?(k)
|
|
@@ -56,7 +57,7 @@ module Datadog
|
|
|
56
57
|
def route_params
|
|
57
58
|
excluded = [:controller, :action]
|
|
58
59
|
|
|
59
|
-
request.env
|
|
60
|
+
request.env.fetch('action_dispatch.request.path_parameters', {}).reject do |k, _v|
|
|
60
61
|
excluded.include?(k)
|
|
61
62
|
end
|
|
62
63
|
end
|