datadog 2.36.0 → 2.37.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -1
  3. data/ext/datadog_profiling_native_extension/clock_id_from_mach.c +5 -12
  4. data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +6 -3
  5. data/ext/datadog_profiling_native_extension/collectors_idle_sampling_helper.c +17 -3
  6. data/ext/datadog_profiling_native_extension/collectors_thread_context.c +219 -98
  7. data/ext/datadog_profiling_native_extension/collectors_thread_context.h +2 -0
  8. data/ext/datadog_profiling_native_extension/extconf.rb +11 -0
  9. data/ext/datadog_profiling_native_extension/ruby_helpers.c +7 -0
  10. data/ext/libdatadog_api/di.c +76 -0
  11. data/ext/libdatadog_api/extconf.rb +6 -0
  12. data/ext/libdatadog_extconf_helpers.rb +1 -1
  13. data/lib/datadog/ai_guard/api_client.rb +6 -4
  14. data/lib/datadog/appsec/api_security/route_extractor.rb +7 -4
  15. data/lib/datadog/appsec/configuration.rb +6 -0
  16. data/lib/datadog/appsec/context.rb +4 -2
  17. data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb +59 -4
  18. data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb +59 -5
  19. data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb +1 -1
  20. data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +4 -0
  21. data/lib/datadog/appsec/contrib/rack/input_peeker.rb +4 -17
  22. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +21 -0
  23. data/lib/datadog/appsec/contrib/rails/gateway/request.rb +2 -1
  24. data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb +60 -7
  25. data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +4 -0
  26. data/lib/datadog/appsec/ext.rb +1 -0
  27. data/lib/datadog/appsec/metrics/collector.rb +18 -1
  28. data/lib/datadog/appsec/metrics/exporter.rb +9 -3
  29. data/lib/datadog/appsec/remote.rb +9 -3
  30. data/lib/datadog/appsec/route_normalizer/rails_route_pattern.rb +176 -0
  31. data/lib/datadog/appsec/route_normalizer/route_pattern.rb +378 -0
  32. data/lib/datadog/appsec/route_normalizer/route_text.rb +57 -0
  33. data/lib/datadog/appsec/route_normalizer.rb +80 -0
  34. data/lib/datadog/appsec/utils/http/body_reader.rb +61 -0
  35. data/lib/datadog/core/configuration/components.rb +82 -2
  36. data/lib/datadog/core/configuration/components_state.rb +6 -1
  37. data/lib/datadog/core/configuration/settings.rb +0 -3
  38. data/lib/datadog/core/configuration/supported_configurations.rb +2 -0
  39. data/lib/datadog/core/evp.rb +11 -0
  40. data/lib/datadog/core/remote/client/capabilities.rb +34 -7
  41. data/lib/datadog/core/telemetry/event/app_started.rb +8 -1
  42. data/lib/datadog/di/base.rb +3 -1
  43. data/lib/datadog/di/code_tracker.rb +5 -2
  44. data/lib/datadog/di/component.rb +110 -36
  45. data/lib/datadog/di/error.rb +10 -0
  46. data/lib/datadog/di/fatal_exceptions.rb +26 -0
  47. data/lib/datadog/di/instrumenter.rb +381 -165
  48. data/lib/datadog/di/probe.rb +14 -0
  49. data/lib/datadog/di/probe_file_loader.rb +8 -3
  50. data/lib/datadog/di/probe_manager.rb +67 -9
  51. data/lib/datadog/di/probe_notification_builder.rb +4 -1
  52. data/lib/datadog/di/probe_notifier_worker.rb +52 -32
  53. data/lib/datadog/di/redactor.rb +16 -1
  54. data/lib/datadog/di/remote.rb +175 -14
  55. data/lib/datadog/di/serializer.rb +12 -5
  56. data/lib/datadog/di/transport/input.rb +8 -4
  57. data/lib/datadog/di.rb +63 -0
  58. data/lib/datadog/kit/appsec/events/v2.rb +60 -2
  59. data/lib/datadog/open_feature/component.rb +29 -6
  60. data/lib/datadog/open_feature/configuration.rb +8 -0
  61. data/lib/datadog/open_feature/ext.rb +2 -0
  62. data/lib/datadog/open_feature/flag_evaluation/aggregator.rb +286 -0
  63. data/lib/datadog/open_feature/flag_evaluation/writer.rb +433 -0
  64. data/lib/datadog/open_feature/hooks/flag_eval_evp_hook.rb +101 -0
  65. data/lib/datadog/open_feature/hooks/{flag_eval_hook.rb → flag_eval_metrics_hook.rb} +1 -1
  66. data/lib/datadog/open_feature/native_evaluator.rb +22 -0
  67. data/lib/datadog/open_feature/provider.rb +59 -27
  68. data/lib/datadog/open_feature/transport.rb +55 -1
  69. data/lib/datadog/opentelemetry/sdk/logs_exporter.rb +5 -12
  70. data/lib/datadog/opentelemetry/sdk/metrics_exporter.rb +4 -11
  71. data/lib/datadog/opentelemetry/sdk.rb +19 -0
  72. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +1 -1
  73. data/lib/datadog/profiling/collectors/idle_sampling_helper.rb +5 -3
  74. data/lib/datadog/symbol_database/component.rb +164 -47
  75. data/lib/datadog/symbol_database/configuration.rb +12 -4
  76. data/lib/datadog/symbol_database/extensions.rb +19 -0
  77. data/lib/datadog/symbol_database/extractor.rb +124 -87
  78. data/lib/datadog/symbol_database/file_hash.rb +3 -1
  79. data/lib/datadog/symbol_database/remote.rb +8 -3
  80. data/lib/datadog/symbol_database/scope_batcher.rb +7 -3
  81. data/lib/datadog/symbol_database/uploader.rb +9 -2
  82. data/lib/datadog/symbol_database.rb +22 -0
  83. data/lib/datadog/tracing/distributed/baggage.rb +2 -1
  84. data/lib/datadog/tracing/ext.rb +9 -0
  85. data/lib/datadog/tracing/remote.rb +34 -3
  86. data/lib/datadog/version.rb +1 -1
  87. data/lib/datadog.rb +1 -0
  88. metadata +19 -6
@@ -21,6 +21,8 @@ VALUE enforce_thread_context_collector_instance(VALUE object);
21
21
  void thread_context_collector_stats(VALUE self_instance, VALUE stats_hash);
22
22
  void thread_context_collector_stats_reset_not_thread_safe(VALUE self_instance);
23
23
  void thread_context_collector_on_serialize(VALUE self_instance);
24
+ void thread_context_collector_profiler_internal_thread_started(void);
25
+ void thread_context_collector_profiler_internal_thread_done(VALUE self_instance);
24
26
 
25
27
  #ifndef NO_GVL_INSTRUMENTATION
26
28
  typedef enum {
@@ -83,6 +83,12 @@ Logging.message("[datadog] End of compiler information\n")
83
83
  # But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
84
84
  append_cflags "-Werror" if ENV["DATADOG_GEM_CI"] == "true"
85
85
 
86
+ # TEMPORARY STOPGAP: libdatadog v36's vendored `common.h` ships duplicate typedefs
87
+ # (a header-dedup regression), which `-Werror` turns into fatal
88
+ # `-Wtypedef-redefinition` errors under C11/gnu11. Keep it a warning until a fixed
89
+ # header lands upstream in libdatadog/libdatadog-rb. Remove this once that ships.
90
+ append_cflags "-Wno-error=typedef-redefinition" if ENV["DATADOG_GEM_CI"] == "true"
91
+
86
92
  # Older gcc releases may not default to C99 and we need to ask for this. This is also used:
87
93
  # * by upstream Ruby -- search for gnu99 in the codebase
88
94
  # * by msgpack, another datadog gem dependency
@@ -154,6 +160,11 @@ $defs << "-DNO_IMEMO_OBJECT_ID" unless RUBY_VERSION < "4"
154
160
  # (see https://bugs.ruby-lang.org/issues/21710)
155
161
  $defs << "-DUSE_DEFERRED_HEAP_ALLOCATION_RECORDING" unless RUBY_VERSION < "4"
156
162
 
163
+ # On Ruby 4.0, we've seen crashes when computing the memsize of a class/module/iclass:
164
+ # rb_obj_memsize_of walks the per-namespace class extensions (classext_memsize), which seem to sometimes be in an inconsistent state
165
+ # (see https://github.com/DataDog/dd-trace-rb/issues/5936)
166
+ $defs << "-DNO_SAFE_CLASS_MEMSIZE" unless RUBY_VERSION < "4"
167
+
157
168
  # This symbol is exclusively visible on certain Ruby versions: 2.6 to 3.2, as well as 3.4 (but not 4.0+)
158
169
  # It's only used to get extra information about an object when a failure happens, so it's a "very nice to have" but not
159
170
  # actually required for correct behavior of the profiler.
@@ -162,9 +162,16 @@ size_t rb_obj_memsize_of(VALUE obj);
162
162
  size_t ruby_obj_memsize_of(VALUE obj) {
163
163
  switch (rb_type(obj)) {
164
164
  case T_OBJECT:
165
+ // On Ruby 4.0, computing the size of a class/module/iclass seems to not be safe: `rb_obj_memsize_of` walks the
166
+ // per-namespace class extensions (`rb_class_classext_foreach` -> `classext_memsize` -> `rb_id_table_memsize`)
167
+ // and can crash the VM when called on objects tracked by heap profiling.
168
+ // See https://github.com/DataDog/dd-trace-rb/issues/5936. Class objects contribute negligibly to heap size,
169
+ // so we skip them (fall through to the `default` branch, returning 0) rather than risk a SIGSEGV.
170
+ #ifndef NO_SAFE_CLASS_MEMSIZE
165
171
  case T_MODULE:
166
172
  case T_CLASS:
167
173
  case T_ICLASS:
174
+ #endif
168
175
  case T_STRING:
169
176
  case T_ARRAY:
170
177
  case T_HASH:
@@ -21,6 +21,12 @@ void rb_objspace_each_objects(
21
21
  // from standard library exception classes like NameError.
22
22
  static ID id_mesg;
23
23
 
24
+ // ID for the fiber-local key that backs the method-probe re-entrancy guard.
25
+ // Storage is the same hashtable that backs Thread#[] / Thread#[]=, but accessed
26
+ // directly via rb_thread_local_aref / rb_thread_local_aset so that user-installed
27
+ // method probes on Thread#[] / Thread#[]= cannot intercept guard reads/writes.
28
+ static ID id_datadog_di_in_probe;
29
+
24
30
  // Returns whether the argument is an IMEMO of type ISEQ.
25
31
  static bool ddtrace_imemo_iseq_p(VALUE v) {
26
32
  return rb_objspace_internal_object_p(v) && RB_TYPE_P(v, T_IMEMO) && ddtrace_imemo_type(v) == IMEMO_TYPE_ISEQ;
@@ -75,6 +81,71 @@ static VALUE exception_message(DDTRACE_UNUSED VALUE _self, VALUE exception) {
75
81
  return rb_ivar_get(exception, id_mesg);
76
82
  }
77
83
 
84
+ /*
85
+ * call-seq:
86
+ * DI.in_probe? -> true | false
87
+ *
88
+ * Returns whether the current fiber is currently inside DI probe processing.
89
+ * Reads the same fiber-local storage as Thread.current[:datadog_di_in_probe]
90
+ * but bypasses Thread#[] method dispatch — a user method probe on Thread#[]
91
+ * cannot observe or intercept this call.
92
+ *
93
+ * @api private
94
+ */
95
+ static VALUE in_probe_p(DDTRACE_UNUSED VALUE _self) {
96
+ VALUE v = rb_thread_local_aref(rb_thread_current(), id_datadog_di_in_probe);
97
+ return RTEST(v) ? Qtrue : Qfalse;
98
+ }
99
+
100
+ /*
101
+ * call-seq:
102
+ * DI.enter_probe -> nil
103
+ *
104
+ * Marks the current fiber as inside DI probe processing. Writes to the same
105
+ * fiber-local storage as Thread.current[:datadog_di_in_probe] = true, but
106
+ * bypasses Thread#[]= method dispatch — a user method probe on Thread#[]=
107
+ * cannot observe or intercept this call.
108
+ *
109
+ * @api private
110
+ */
111
+ static VALUE enter_probe(DDTRACE_UNUSED VALUE _self) {
112
+ rb_thread_local_aset(rb_thread_current(), id_datadog_di_in_probe, Qtrue);
113
+ return Qnil;
114
+ }
115
+
116
+ /*
117
+ * call-seq:
118
+ * DI.leave_probe -> nil
119
+ *
120
+ * Marks the current fiber as no longer inside DI probe processing. Writes to
121
+ * the same fiber-local storage as Thread.current[:datadog_di_in_probe] = nil,
122
+ * but bypasses Thread#[]= method dispatch — a user method probe on Thread#[]=
123
+ * cannot observe or intercept this call.
124
+ *
125
+ * @api private
126
+ */
127
+ static VALUE leave_probe(DDTRACE_UNUSED VALUE _self) {
128
+ rb_thread_local_aset(rb_thread_current(), id_datadog_di_in_probe, Qnil);
129
+ return Qnil;
130
+ }
131
+
132
+ /*
133
+ * call-seq:
134
+ * DI.hash?(obj) -> true | false
135
+ *
136
+ * Returns whether the given object is a Hash via a direct type check
137
+ * (RB_TYPE_P), bypassing Kernel#is_a? method dispatch. Used in the method
138
+ * probe wrapper (the Ruby < 3 argument-serialization split) to test whether
139
+ * the trailing argument is a hash without giving a user-installed method
140
+ * probe on Kernel#is_a? a chance to recurse. Matches Hash and its
141
+ * subclasses, like is_a?(Hash), since their instances are still T_HASH.
142
+ *
143
+ * @api private
144
+ */
145
+ static VALUE is_hash(DDTRACE_UNUSED VALUE _self, VALUE obj) {
146
+ return RB_TYPE_P(obj, T_HASH) ? Qtrue : Qfalse;
147
+ }
148
+
78
149
  // rb_iseq_type was added in Ruby 3.1 (commit 89a02d89 by Koichi Sasada,
79
150
  // 2021-12-19). It returns the iseq type as a Symbol. On Ruby < 3.1 this
80
151
  // function does not exist, so have_func('rb_iseq_type') in extconf.rb
@@ -117,10 +188,15 @@ static VALUE iseq_type(DDTRACE_UNUSED VALUE _self, VALUE iseq_val) {
117
188
 
118
189
  void di_init(VALUE datadog_module) {
119
190
  id_mesg = rb_intern("mesg");
191
+ id_datadog_di_in_probe = rb_intern("datadog_di_in_probe");
120
192
 
121
193
  VALUE di_module = rb_define_module_under(datadog_module, "DI");
122
194
  rb_define_singleton_method(di_module, "all_iseqs", all_iseqs, 0);
123
195
  rb_define_singleton_method(di_module, "exception_message", exception_message, 1);
196
+ rb_define_singleton_method(di_module, "in_probe?", in_probe_p, 0);
197
+ rb_define_singleton_method(di_module, "enter_probe", enter_probe, 0);
198
+ rb_define_singleton_method(di_module, "leave_probe", leave_probe, 0);
199
+ rb_define_singleton_method(di_module, "hash?", is_hash, 1);
124
200
  #ifdef HAVE_RB_ISEQ_TYPE
125
201
  rb_define_singleton_method(di_module, "iseq_type", iseq_type, 1);
126
202
  #endif
@@ -41,6 +41,12 @@ Datadog::LibdatadogExtconfHelpers.dump_mkmf_log_on_failure!
41
41
  # But we can enable it in CI, so that we quickly spot any new warnings that just got introduced.
42
42
  append_cflags '-Werror' if ENV['DATADOG_GEM_CI'] == 'true'
43
43
 
44
+ # TEMPORARY STOPGAP: libdatadog v36's vendored `common.h` ships duplicate typedefs
45
+ # (a header-dedup regression), which `-Werror` turns into fatal
46
+ # `-Wtypedef-redefinition` errors under C11/gnu11. Keep it a warning until a fixed
47
+ # header lands upstream in libdatadog/libdatadog-rb. Remove this once that ships.
48
+ append_cflags '-Wno-error=typedef-redefinition' if ENV['DATADOG_GEM_CI'] == 'true'
49
+
44
50
  # Older gcc releases may not default to C99 and we need to ask for this. This is also used:
45
51
  # * by upstream Ruby -- search for gnu99 in the codebase
46
52
  # * by msgpack, another datadog gem dependency
@@ -10,7 +10,7 @@ module Datadog
10
10
  module LibdatadogExtconfHelpers
11
11
  # Used to make sure the correct gem version gets loaded, as extconf.rb does not get run with "bundle exec" and thus
12
12
  # may see multiple libdatadog versions. See https://github.com/DataDog/dd-trace-rb/pull/2531 for the horror story.
13
- LIBDATADOG_VERSION = '~> 35.0.0.1.0'
13
+ LIBDATADOG_VERSION = '~> 36.0.0.1.0'
14
14
 
15
15
  # Used as an workaround for a limitation with how dynamic linking works in environments where the datadog gem and
16
16
  # libdatadog are moved after the extension gets compiled.
@@ -17,11 +17,13 @@ module Datadog
17
17
 
18
18
  @endpoint_uri = if endpoint
19
19
  URI(endpoint) #: URI::HTTP
20
+ elsif Datadog.configuration.site
21
+ host = Datadog.configuration.site.dup
22
+ host.prepend("app.") if host.count(".") == 1
23
+
24
+ URI::HTTPS.build(host: host, path: DEFAULT_PATH)
20
25
  else
21
- URI::HTTPS.build(
22
- host: Datadog.configuration.site || DEFAULT_SITE,
23
- path: DEFAULT_PATH
24
- )
26
+ URI::HTTPS.build(host: DEFAULT_SITE, path: DEFAULT_PATH)
25
27
  end
26
28
 
27
29
  @headers = {
@@ -31,16 +31,19 @@ module Datadog
31
31
  # uses `sinatra.route` with a string like "GET /users/:id"
32
32
  # Grape
33
33
  # uses `grape.routing_args` with a hash with a `:route_info` key
34
- # that contains a `Grape::Router::Route` object that contains
35
- # `Grape::Router::Pattern` object with an `origin` method
34
+ # that contains a {Grape::Router::Route} object that contains
35
+ # {Grape::Router::Pattern} object with an `origin` method
36
+ # Rails with `action_pack` tracing contrib (fast path)
37
+ # `datadog.action_dispatch.route` stores the {Journey::Route}
38
+ # object set by the tracer at routing time
36
39
  # Rails < 7.1 (slow path)
37
- # uses `action_dispatch.routes` to store `ActionDispatch::Routing::RouteSet`
40
+ # uses `action_dispatch.routes` to store {ActionDispatch::Routing::RouteSet}
38
41
  # which can recognize requests
39
42
  # Rails > 7.1 (fast path)
40
43
  # uses `action_dispatch.route_uri_pattern` with a string like
41
44
  # "/users/:id(.:format)"
42
45
  # Rails > 8.1.1 (fast path)
43
- # uses `action_dispatch.route` to store the ActionDispatch::Journey::Route
46
+ # uses `action_dispatch.route` to store the {ActionDispatch::Journey::Route}
44
47
  # that matched when the request was routed
45
48
  #
46
49
  # WARNING: This method works only *after* the request has been routed.
@@ -414,6 +414,12 @@ module Datadog
414
414
  o.env 'DD_API_SECURITY_MAX_DOWNSTREAM_REQUEST_BODY_ANALYSIS'
415
415
  o.default 1
416
416
  end
417
+
418
+ option :max_downstream_body_bytes do |o|
419
+ o.type :int
420
+ o.env 'DD_API_SECURITY_MAX_DOWNSTREAM_BODY_BYTES'
421
+ o.default 10_485_760
422
+ end
417
423
  end
418
424
  end
419
425
 
@@ -20,7 +20,7 @@ module Datadog
20
20
  ActiveContextError = Class.new(StandardError) # steep:ignore IncompatibleAssignment
21
21
 
22
22
  # TODO: add delegators for active trace span
23
- attr_reader :trace, :span
23
+ attr_reader :trace, :span, :metrics
24
24
 
25
25
  # Shared mutable storage for counters, flags, and data accumulated during
26
26
  # the request's lifecycle.
@@ -103,7 +103,8 @@ module Datadog
103
103
  end
104
104
 
105
105
  def extract_schema!
106
- waf_result = @waf_runner.run({'waf.context.processor' => {'extract-schema' => true}}, {})
106
+ persistent_data = {'waf.context.processor' => {'extract-schema' => true}}
107
+ waf_result = run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
107
108
  security_event = AppSec::SecurityEvent.new(waf_result, trace: trace, span: span)
108
109
 
109
110
  @state[:schema_extracted] = security_event.schema?
@@ -116,6 +117,7 @@ module Datadog
116
117
 
117
118
  Metrics::Exporter.export_waf_metrics(@metrics.waf, @span)
118
119
  Metrics::Exporter.export_rasp_metrics(@metrics.rasp, @span)
120
+ Metrics::Exporter.export_downstream_response_metrics(@metrics.downstream_responses, @span)
119
121
  end
120
122
 
121
123
  def export_request_telemetry
@@ -7,6 +7,8 @@ require_relative '../../trace_keeper'
7
7
  require_relative '../../security_event'
8
8
  require_relative '../../utils/http/url_encoded'
9
9
  require_relative '../../utils/http/body'
10
+ require_relative '../../utils/http/body_reader'
11
+ require_relative '../../utils/http/media_type'
10
12
 
11
13
  module Datadog
12
14
  module AppSec
@@ -39,7 +41,7 @@ module Datadog
39
41
  end
40
42
 
41
43
  if !is_redirect && data[SAMPLE_BODY_KEY]
42
- body = parse_body(data[:body], content_type: headers['content-type'])
44
+ body = parse_request_body(data[:body], content_type: headers['content-type'])
43
45
  ephemeral_data['server.io.net.request.body'] = body if body
44
46
  end
45
47
 
@@ -67,7 +69,7 @@ module Datadog
67
69
  end
68
70
 
69
71
  if !is_redirect && data[SAMPLE_BODY_KEY]
70
- body = parse_body(data.dig(:response, :body), content_type: headers['content-type'])
72
+ body = parse_response_body(data.dig(:response, :body), headers: headers, context: context)
71
73
  ephemeral_data['server.io.net.response.body'] = body if body
72
74
  end
73
75
 
@@ -89,11 +91,64 @@ module Datadog
89
91
  data[SAMPLE_BODY_KEY] = true
90
92
  end
91
93
 
92
- def parse_body(body, content_type:)
94
+ def parse_request_body(body, content_type:)
93
95
  media_type = Utils::HTTP::MediaType.parse(content_type)
94
96
  return unless media_type
95
97
 
96
- Utils::HTTP::Body.parse(body, media_type: media_type)
98
+ limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
99
+ content = read_body(body, limit: limit)
100
+ return if content.nil? || content.bytesize > limit
101
+
102
+ Utils::HTTP::Body.parse(content, media_type: media_type)
103
+ end
104
+
105
+ def parse_response_body(body, headers:, context:)
106
+ return unless readable_body?(body)
107
+
108
+ media_type = Utils::HTTP::MediaType.parse(headers['content-type'])
109
+ if !media_type || media_type.type != 'application'
110
+ context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
111
+ return
112
+ end
113
+
114
+ subtype = media_type.subtype
115
+ if subtype != 'json' && !subtype.end_with?('+json') && subtype != 'x-www-form-urlencoded'
116
+ context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
117
+ return
118
+ end
119
+
120
+ content_length_value = headers['content-length']
121
+ if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
122
+ context.metrics.record_ignored_downstream_response_body(:content_length_missing)
123
+ return
124
+ end
125
+
126
+ content_length = content_length_value.to_i
127
+ max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
128
+ if content_length > max
129
+ context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
130
+ return
131
+ end
132
+
133
+ content = read_body(body, limit: content_length)
134
+ return if content.nil?
135
+
136
+ if content.bytesize > content_length
137
+ context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
138
+ return
139
+ end
140
+
141
+ Utils::HTTP::Body.parse(content, media_type: media_type)
142
+ end
143
+
144
+ def readable_body?(body)
145
+ body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
146
+ end
147
+
148
+ def read_body(body, limit:)
149
+ Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
150
+ rescue
151
+ nil
97
152
  end
98
153
 
99
154
  def request_url(data)
@@ -3,8 +3,9 @@
3
3
  require_relative '../../event'
4
4
  require_relative '../../trace_keeper'
5
5
  require_relative '../../security_event'
6
- require_relative '../../utils/http/media_type'
7
6
  require_relative '../../utils/http/body'
7
+ require_relative '../../utils/http/body_reader'
8
+ require_relative '../../utils/http/media_type'
8
9
 
9
10
  module Datadog
10
11
  module AppSec
@@ -37,7 +38,7 @@ module Datadog
37
38
  end
38
39
 
39
40
  if !is_redirect && env[SAMPLE_BODY_KEY]
40
- body = parse_body(env.body, content_type: headers['content-type'])
41
+ body = parse_request_body(env.body, content_type: headers['content-type'])
41
42
  ephemeral_data['server.io.net.request.body'] = body if body
42
43
  end
43
44
 
@@ -64,7 +65,7 @@ module Datadog
64
65
  end
65
66
 
66
67
  if !is_redirect && env[SAMPLE_BODY_KEY]
67
- body = parse_body(env.body, content_type: headers['content-type'])
68
+ body = parse_response_body(env.body, headers: headers, context: context)
68
69
  ephemeral_data['server.io.net.response.body'] = body if body
69
70
  end
70
71
 
@@ -82,11 +83,64 @@ module Datadog
82
83
  env[SAMPLE_BODY_KEY] = true
83
84
  end
84
85
 
85
- def parse_body(body, content_type:)
86
+ def parse_request_body(body, content_type:)
86
87
  media_type = Utils::HTTP::MediaType.parse(content_type)
87
88
  return unless media_type
88
89
 
89
- Utils::HTTP::Body.parse(body, media_type: media_type)
90
+ limit = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
91
+ content = read_body(body, limit: limit)
92
+ return if content.nil? || content.bytesize > limit
93
+
94
+ Utils::HTTP::Body.parse(content, media_type: media_type)
95
+ end
96
+
97
+ def parse_response_body(body, headers:, context:)
98
+ return unless readable_body?(body)
99
+
100
+ media_type = Utils::HTTP::MediaType.parse(headers['content-type'])
101
+ if !media_type || media_type.type != 'application'
102
+ context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
103
+ return
104
+ end
105
+
106
+ subtype = media_type.subtype
107
+ if subtype != 'json' && !subtype.end_with?('+json') && subtype != 'x-www-form-urlencoded'
108
+ context.metrics.record_ignored_downstream_response_body(:content_type_invalid)
109
+ return
110
+ end
111
+
112
+ content_length_value = headers['content-length']
113
+ if !content_length_value.is_a?(String) || !content_length_value.match?(/\A[1-9][0-9]*\z/)
114
+ context.metrics.record_ignored_downstream_response_body(:content_length_missing)
115
+ return
116
+ end
117
+
118
+ content_length = content_length_value.to_i
119
+ max = Datadog.configuration.appsec.api_security.downstream_body_analysis.max_downstream_body_bytes
120
+ if content_length > max
121
+ context.metrics.record_ignored_downstream_response_body(:content_length_too_big)
122
+ return
123
+ end
124
+
125
+ content = read_body(body, limit: content_length)
126
+ return if content.nil?
127
+
128
+ if content.bytesize > content_length
129
+ context.metrics.record_ignored_downstream_response_body(:content_exceed_content_length)
130
+ return
131
+ end
132
+
133
+ Utils::HTTP::Body.parse(content, media_type: media_type)
134
+ end
135
+
136
+ def readable_body?(body)
137
+ body.is_a?(String) || (body.respond_to?(:read) && body.respond_to?(:rewind))
138
+ end
139
+
140
+ def read_body(body, limit:)
141
+ Utils::HTTP::BodyReader.read(body, limit: limit, rewind_before_read: true)
142
+ rescue
143
+ nil
90
144
  end
91
145
 
92
146
  def normalize_headers(headers)
@@ -72,11 +72,11 @@ module Datadog
72
72
  args_hash, fragment_name, nil, arguments_from_directives(fragment.directives, query_variables)
73
73
  )
74
74
 
75
+ # re-used fragments are not expanded
75
76
  visited_fragments[fragment_name] = true
76
77
  arguments_from_selections(
77
78
  fragment.selections, query_variables, args_hash, fragments, visited_fragments
78
79
  )
79
- visited_fragments.delete(fragment_name)
80
80
  when ::GraphQL::Language::Nodes::Field
81
81
  selection_name = selection.alias || selection.name
82
82
  field_arguments = arguments_hash(selection.arguments, query_variables) unless selection.arguments.empty?
@@ -104,6 +104,10 @@ module Datadog
104
104
  body = gateway_request.form_hash
105
105
  persistent_data['server.request.body'] = body if body
106
106
  end
107
+ # NOTE: Body was parsed before measurement, keep byte_length unset
108
+ elsif gateway_request.env.key?('rack.request.form_hash')
109
+ body = gateway_request.env['rack.request.form_hash']
110
+ persistent_data['server.request.body'] = body if body
107
111
  end
108
112
 
109
113
  next stack.call(request) if persistent_data.empty?
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'stringio'
4
4
  require_relative 'buffered_input'
5
+ require_relative '../../utils/http/body_reader'
5
6
 
6
7
  module Datadog
7
8
  module AppSec
@@ -38,7 +39,9 @@ module Datadog
38
39
  # in case an upstream framework already consumed part of the stream
39
40
  return if rewindable && !rewind(rack_input)
40
41
 
41
- buffer = peek(rack_input, limit)
42
+ # NOTE: Read one byte past the limit to distinguish an exact-limit body
43
+ # from an over-limit body without reading the whole stream.
44
+ buffer = Utils::HTTP::BodyReader.read_stream(rack_input, limit: limit)
42
45
  over_limit = buffer.bytesize > limit
43
46
 
44
47
  if rewindable
@@ -69,22 +72,6 @@ module Datadog
69
72
  Datadog.logger.debug { "AppSec: Failed to rewind `rack.input`: #{e.class}: #{e.message}" }
70
73
  false
71
74
  end
72
-
73
- private_class_method def peek(io, limit)
74
- # NOTE: Read one byte past the limit to distinguish an exact-limit body
75
- # from an over-limit body without reading the whole stream.
76
- max = limit + 1
77
- buffer = +''.b
78
-
79
- while buffer.bytesize <= limit
80
- chunk = io.read(max - buffer.bytesize)
81
- break if chunk.nil? || chunk.empty?
82
-
83
- buffer << chunk
84
- end
85
-
86
- buffer
87
- end
88
75
  end
89
76
  end
90
77
  end
@@ -10,6 +10,7 @@ require_relative '../../event'
10
10
  require_relative '../../response'
11
11
  require_relative '../../api_security'
12
12
  require_relative '../../default_header_tags'
13
+ require_relative '../../route_normalizer'
13
14
  require_relative '../../security_event'
14
15
  require_relative '../../instrumentation/gateway'
15
16
 
@@ -76,6 +77,8 @@ module Datadog
76
77
  nil
77
78
  end
78
79
 
80
+ add_normalized_route_tag(ctx, env)
81
+
79
82
  if interrupt_params
80
83
  ctx.mark_as_interrupted!
81
84
  http_response = AppSec::Response.from_interrupt_params(interrupt_params, env['HTTP_ACCEPT']).to_rack
@@ -198,6 +201,24 @@ module Datadog
198
201
  end
199
202
  end
200
203
 
204
+ def add_normalized_route_tag(context, env)
205
+ return unless AppSec::APISecurity.enabled?
206
+
207
+ span = context.span
208
+ return unless span
209
+
210
+ pattern = context.trace&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE)
211
+ return unless pattern
212
+
213
+ # NOTE: To build full path that covers mounted engines we need to add
214
+ # pre-computed by Tracer route path tag to the normalized route
215
+ prefix = context.trace&.get_tag(Tracing::Metadata::Ext::HTTP::TAG_ROUTE_PATH) || env['SCRIPT_NAME']
216
+ normalized_route = RouteNormalizer.extract_normalized_route(env, prefix: prefix, pattern: pattern)
217
+ return unless normalized_route
218
+
219
+ span.set_tag(AppSec::Ext::TAG_NORMALIZED_ROUTE, "#{prefix}#{normalized_route}")
220
+ end
221
+
201
222
  def oneshot_tags_sent?
202
223
  @oneshot_tags_sent
203
224
  end
@@ -47,6 +47,7 @@ module Datadog
47
47
  body = request.env['action_dispatch.request.request_parameters']
48
48
 
49
49
  return if body.nil?
50
+ return body unless request.env['action_dispatch.request.path_parameters']
50
51
 
51
52
  body.reject do |k, _v|
52
53
  request.env['action_dispatch.request.path_parameters'].key?(k)
@@ -56,7 +57,7 @@ module Datadog
56
57
  def route_params
57
58
  excluded = [:controller, :action]
58
59
 
59
- request.env['action_dispatch.request.path_parameters'].reject do |k, _v|
60
+ request.env.fetch('action_dispatch.request.path_parameters', {}).reject do |k, _v|
60
61
  excluded.include?(k)
61
62
  end
62
63
  end