ddtrace 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -16
  3. data/CHANGELOG.md +31 -2
  4. data/LICENSE-3rdparty.csv +3 -2
  5. data/README.md +2 -2
  6. data/ddtrace.gemspec +12 -3
  7. data/docs/GettingStarted.md +19 -2
  8. data/docs/ProfilingDevelopment.md +8 -8
  9. data/docs/UpgradeGuide.md +3 -3
  10. data/ext/ddtrace_profiling_loader/ddtrace_profiling_loader.c +118 -0
  11. data/ext/ddtrace_profiling_loader/extconf.rb +53 -0
  12. data/ext/ddtrace_profiling_native_extension/NativeExtensionDesign.md +31 -5
  13. data/ext/ddtrace_profiling_native_extension/clock_id_from_pthread.c +0 -8
  14. data/ext/ddtrace_profiling_native_extension/collectors_stack.c +278 -0
  15. data/ext/ddtrace_profiling_native_extension/extconf.rb +70 -100
  16. data/ext/ddtrace_profiling_native_extension/libddprof_helpers.h +13 -0
  17. data/ext/ddtrace_profiling_native_extension/native_extension_helpers.rb +186 -0
  18. data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c +579 -7
  19. data/ext/ddtrace_profiling_native_extension/private_vm_api_access.h +30 -0
  20. data/ext/ddtrace_profiling_native_extension/profiling.c +7 -0
  21. data/ext/ddtrace_profiling_native_extension/stack_recorder.c +139 -0
  22. data/ext/ddtrace_profiling_native_extension/stack_recorder.h +28 -0
  23. data/lib/datadog/appsec/autoload.rb +2 -2
  24. data/lib/datadog/appsec/configuration/settings.rb +19 -0
  25. data/lib/datadog/appsec/configuration.rb +8 -0
  26. data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +76 -33
  27. data/lib/datadog/appsec/contrib/rack/integration.rb +1 -0
  28. data/lib/datadog/appsec/contrib/rack/patcher.rb +0 -1
  29. data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb +64 -0
  30. data/lib/datadog/appsec/contrib/rack/request.rb +6 -0
  31. data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb +41 -0
  32. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +60 -5
  33. data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +81 -0
  34. data/lib/datadog/appsec/contrib/rails/patcher.rb +34 -1
  35. data/lib/datadog/appsec/contrib/rails/reactive/action.rb +68 -0
  36. data/lib/datadog/appsec/contrib/rails/request.rb +33 -0
  37. data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +124 -0
  38. data/lib/datadog/appsec/contrib/sinatra/patcher.rb +69 -2
  39. data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb +63 -0
  40. data/lib/datadog/appsec/event.rb +33 -18
  41. data/lib/datadog/appsec/extensions.rb +0 -3
  42. data/lib/datadog/appsec/processor.rb +45 -2
  43. data/lib/datadog/appsec/rate_limiter.rb +5 -0
  44. data/lib/datadog/appsec/reactive/operation.rb +0 -1
  45. data/lib/datadog/ci/ext/environment.rb +21 -7
  46. data/lib/datadog/core/configuration/agent_settings_resolver.rb +1 -1
  47. data/lib/datadog/core/configuration/components.rb +22 -4
  48. data/lib/datadog/core/configuration/settings.rb +3 -3
  49. data/lib/datadog/core/configuration.rb +7 -5
  50. data/lib/datadog/core/environment/cgroup.rb +3 -1
  51. data/lib/datadog/core/environment/container.rb +2 -1
  52. data/lib/datadog/core/environment/variable_helpers.rb +26 -2
  53. data/lib/datadog/core/logging/ext.rb +11 -0
  54. data/lib/datadog/core/metrics/client.rb +15 -5
  55. data/lib/datadog/core/runtime/metrics.rb +1 -1
  56. data/lib/datadog/core/workers/async.rb +3 -1
  57. data/lib/datadog/core/workers/runtime_metrics.rb +0 -3
  58. data/lib/datadog/core.rb +6 -0
  59. data/lib/datadog/kit/enable_core_dumps.rb +50 -0
  60. data/lib/datadog/kit/identity.rb +63 -0
  61. data/lib/datadog/kit.rb +11 -0
  62. data/lib/datadog/opentracer/tracer.rb +0 -2
  63. data/lib/datadog/profiling/collectors/old_stack.rb +298 -0
  64. data/lib/datadog/profiling/collectors/stack.rb +6 -287
  65. data/lib/datadog/profiling/encoding/profile.rb +0 -1
  66. data/lib/datadog/profiling/ext.rb +1 -1
  67. data/lib/datadog/profiling/flush.rb +1 -1
  68. data/lib/datadog/profiling/load_native_extension.rb +22 -0
  69. data/lib/datadog/profiling/recorder.rb +1 -1
  70. data/lib/datadog/profiling/scheduler.rb +1 -1
  71. data/lib/datadog/profiling/stack_recorder.rb +33 -0
  72. data/lib/datadog/profiling/tag_builder.rb +48 -0
  73. data/lib/datadog/profiling/tasks/exec.rb +2 -2
  74. data/lib/datadog/profiling/tasks/setup.rb +6 -4
  75. data/lib/datadog/profiling.rb +29 -27
  76. data/lib/datadog/tracing/buffer.rb +9 -3
  77. data/lib/datadog/tracing/contrib/action_view/patcher.rb +0 -1
  78. data/lib/datadog/tracing/contrib/active_record/configuration/resolver.rb +2 -2
  79. data/lib/datadog/tracing/contrib/active_record/utils.rb +1 -1
  80. data/lib/datadog/tracing/contrib/active_record/vendor/connection_specification.rb +1 -1
  81. data/lib/datadog/tracing/contrib/active_support/notifications/subscription.rb +4 -2
  82. data/lib/datadog/tracing/contrib/concurrent_ruby/context_composite_executor_service.rb +10 -3
  83. data/lib/datadog/tracing/contrib/dalli/patcher.rb +0 -1
  84. data/lib/datadog/tracing/contrib/delayed_job/patcher.rb +0 -1
  85. data/lib/datadog/tracing/contrib/elasticsearch/integration.rb +9 -3
  86. data/lib/datadog/tracing/contrib/elasticsearch/patcher.rb +38 -2
  87. data/lib/datadog/tracing/contrib/ethon/patcher.rb +0 -1
  88. data/lib/datadog/tracing/contrib/extensions.rb +0 -2
  89. data/lib/datadog/tracing/contrib/faraday/patcher.rb +0 -1
  90. data/lib/datadog/tracing/contrib/grape/patcher.rb +0 -1
  91. data/lib/datadog/tracing/contrib/graphql/patcher.rb +0 -1
  92. data/lib/datadog/tracing/contrib/grpc/patcher.rb +0 -1
  93. data/lib/datadog/tracing/contrib/kafka/patcher.rb +0 -1
  94. data/lib/datadog/tracing/contrib/lograge/instrumentation.rb +2 -1
  95. data/lib/datadog/tracing/contrib/qless/patcher.rb +0 -1
  96. data/lib/datadog/tracing/contrib/que/patcher.rb +0 -1
  97. data/lib/datadog/tracing/contrib/racecar/patcher.rb +0 -1
  98. data/lib/datadog/tracing/contrib/rails/log_injection.rb +3 -16
  99. data/lib/datadog/tracing/contrib/rake/instrumentation.rb +2 -2
  100. data/lib/datadog/tracing/contrib/rake/patcher.rb +0 -1
  101. data/lib/datadog/tracing/contrib/redis/patcher.rb +0 -1
  102. data/lib/datadog/tracing/contrib/resque/patcher.rb +0 -1
  103. data/lib/datadog/tracing/contrib/rest_client/patcher.rb +0 -1
  104. data/lib/datadog/tracing/contrib/semantic_logger/instrumentation.rb +2 -1
  105. data/lib/datadog/tracing/contrib/sidekiq/configuration/settings.rb +1 -0
  106. data/lib/datadog/tracing/contrib/sidekiq/server_tracer.rb +20 -1
  107. data/lib/datadog/tracing/contrib/sinatra/framework.rb +11 -0
  108. data/lib/datadog/tracing/contrib/sinatra/patcher.rb +0 -1
  109. data/lib/datadog/tracing/contrib/sneakers/patcher.rb +0 -1
  110. data/lib/datadog/tracing/contrib/sucker_punch/patcher.rb +0 -1
  111. data/lib/datadog/tracing/event.rb +2 -1
  112. data/lib/datadog/tracing/sampling/priority_sampler.rb +4 -5
  113. data/lib/datadog/tracing/sampling/rule.rb +12 -6
  114. data/lib/datadog/tracing/sampling/rule_sampler.rb +3 -5
  115. data/lib/datadog/tracing/span_operation.rb +2 -3
  116. data/lib/datadog/tracing/trace_operation.rb +0 -1
  117. data/lib/ddtrace/transport/http/client.rb +2 -1
  118. data/lib/ddtrace/transport/http/response.rb +34 -4
  119. data/lib/ddtrace/transport/io/client.rb +3 -1
  120. data/lib/ddtrace/version.rb +1 -1
  121. data/lib/ddtrace.rb +1 -0
  122. metadata +43 -6
@@ -0,0 +1,139 @@
1
+ #include <ruby.h>
2
+ #include <ruby/thread.h>
3
+ #include "stack_recorder.h"
4
+ #include "libddprof_helpers.h"
5
+
6
+ // Used to wrap a ddprof_ffi_Profile in a Ruby object and expose Ruby-level serialization APIs
7
+ // This file implements the native bits of the Datadog::Profiling::StackRecorder class
8
+
9
+ static VALUE ok_symbol = Qnil; // :ok in Ruby
10
+ static VALUE error_symbol = Qnil; // :error in Ruby
11
+
12
+ static ID ruby_time_from_id; // id of :ruby_time_from in Ruby
13
+
14
+ static VALUE stack_recorder_class = Qnil;
15
+
16
+ struct call_serialize_without_gvl_arguments {
17
+ ddprof_ffi_Profile *profile;
18
+ ddprof_ffi_SerializeResult result;
19
+ bool serialize_ran;
20
+ };
21
+
22
+ static VALUE _native_new(VALUE klass);
23
+ static void stack_recorder_typed_data_free(void *data);
24
+ static VALUE _native_serialize(VALUE self, VALUE recorder_instance);
25
+ static VALUE ruby_time_from(ddprof_ffi_Timespec ddprof_time);
26
+ static void *call_serialize_without_gvl(void *call_args);
27
+
28
+ void stack_recorder_init(VALUE profiling_module) {
29
+ stack_recorder_class = rb_define_class_under(profiling_module, "StackRecorder", rb_cObject);
30
+
31
+ // Instances of the StackRecorder class are going to be "TypedData" objects.
32
+ // "TypedData" objects are special objects in the Ruby VM that can wrap C structs.
33
+ // In our case, we're going to keep a libddprof profile reference inside our object.
34
+ //
35
+ // Because Ruby doesn't know how to initialize libddprof profiles, we MUST override the allocation function for objects
36
+ // of this class so that we can manage this part. Not overriding or disabling the allocation function is a common
37
+ // gotcha for "TypedData" objects that can very easily lead to VM crashes, see for instance
38
+ // https://bugs.ruby-lang.org/issues/18007 for a discussion around this.
39
+ rb_define_alloc_func(stack_recorder_class, _native_new);
40
+
41
+ rb_define_singleton_method(stack_recorder_class, "_native_serialize", _native_serialize, 1);
42
+
43
+ ok_symbol = ID2SYM(rb_intern_const("ok"));
44
+ error_symbol = ID2SYM(rb_intern_const("error"));
45
+ ruby_time_from_id = rb_intern_const("ruby_time_from");
46
+ }
47
+
48
+ // This structure is used to define a Ruby object that stores a pointer to a ddprof_ffi_Profile instance
49
+ // See also https://github.com/ruby/ruby/blob/master/doc/extension.rdoc for how this works
50
+ static const rb_data_type_t stack_recorder_typed_data = {
51
+ .wrap_struct_name = "Datadog::Profiling::StackRecorder",
52
+ .function = {
53
+ .dfree = stack_recorder_typed_data_free,
54
+ .dsize = NULL, // We don't track profile memory usage (although it'd be cool if we did!)
55
+ // No need to provide dmark nor dcompact because we don't directly reference Ruby VALUEs from inside this object
56
+ },
57
+ .flags = RUBY_TYPED_FREE_IMMEDIATELY
58
+ };
59
+
60
+ static VALUE _native_new(VALUE klass) {
61
+ ddprof_ffi_Slice_value_type sample_types = {.ptr = enabled_value_types, .len = ENABLED_VALUE_TYPES_COUNT};
62
+
63
+ ddprof_ffi_Profile *profile = ddprof_ffi_Profile_new(sample_types, NULL /* Period is optional */);
64
+
65
+ return TypedData_Wrap_Struct(klass, &stack_recorder_typed_data, profile);
66
+ }
67
+
68
+ static void stack_recorder_typed_data_free(void *data) {
69
+ ddprof_ffi_Profile_free((ddprof_ffi_Profile *) data);
70
+ }
71
+
72
+ static VALUE _native_serialize(VALUE self, VALUE recorder_instance) {
73
+ ddprof_ffi_Profile *profile;
74
+ TypedData_Get_Struct(recorder_instance, ddprof_ffi_Profile, &stack_recorder_typed_data, profile);
75
+
76
+ // We'll release the Global VM Lock while we're calling serialize, so that the Ruby VM can continue to work while this
77
+ // is pending
78
+ struct call_serialize_without_gvl_arguments args = {.profile = profile, .serialize_ran = false};
79
+
80
+ // We use rb_thread_call_without_gvl2 for similar reasons as in http_transport.c: we don't want pending interrupts
81
+ // that cause exceptions to be raised to be processed as otherwise we can leak the serialized profile.
82
+ rb_thread_call_without_gvl2(call_serialize_without_gvl, &args, /* No interruption supported */ NULL, NULL);
83
+
84
+ // This weird corner case can happen if rb_thread_call_without_gvl2 returns immediately due to an interrupt
85
+ // without ever calling call_serialize_without_gvl. In this situation, we don't have anything to clean up, we can
86
+ // just return.
87
+ if (!args.serialize_ran) {
88
+ return rb_ary_new_from_args(2, error_symbol, rb_str_new_cstr("Interrupted before call_serialize_without_gvl ran"));
89
+ }
90
+
91
+ ddprof_ffi_SerializeResult serialized_profile = args.result;
92
+
93
+ if (serialized_profile.tag == DDPROF_FFI_SERIALIZE_RESULT_ERR) {
94
+ VALUE err_details = ruby_string_from_vec_u8(serialized_profile.err);
95
+ ddprof_ffi_SerializeResult_drop(serialized_profile);
96
+ return rb_ary_new_from_args(2, error_symbol, err_details);
97
+ }
98
+
99
+ VALUE encoded_pprof = ruby_string_from_vec_u8(serialized_profile.ok.buffer);
100
+
101
+ ddprof_ffi_Timespec ddprof_start = serialized_profile.ok.start;
102
+ ddprof_ffi_Timespec ddprof_finish = serialized_profile.ok.end;
103
+
104
+ // Clean up libddprof object to avoid leaking in case ruby_time_from raises an exception
105
+ ddprof_ffi_SerializeResult_drop(serialized_profile);
106
+
107
+ VALUE start = ruby_time_from(ddprof_start);
108
+ VALUE finish = ruby_time_from(ddprof_finish);
109
+
110
+ if (!ddprof_ffi_Profile_reset(profile)) return rb_ary_new_from_args(2, error_symbol, rb_str_new_cstr("Failed to reset profile"));
111
+
112
+ return rb_ary_new_from_args(2, ok_symbol, rb_ary_new_from_args(3, start, finish, encoded_pprof));
113
+ }
114
+
115
+ static VALUE ruby_time_from(ddprof_ffi_Timespec ddprof_time) {
116
+ #ifndef NO_RB_TIME_TIMESPEC_NEW // Modern Rubies
117
+ const int utc = INT_MAX - 1; // From Ruby sources
118
+ struct timespec time = {.tv_sec = ddprof_time.seconds, .tv_nsec = ddprof_time.nanoseconds};
119
+ return rb_time_timespec_new(&time, utc);
120
+ #else // Ruby < 2.3
121
+ return rb_funcall(stack_recorder_class, ruby_time_from_id, 2, LONG2NUM(ddprof_time.seconds), UINT2NUM(ddprof_time.nanoseconds));
122
+ #endif
123
+ }
124
+
125
+ void record_sample(VALUE recorder_instance, ddprof_ffi_Sample sample) {
126
+ ddprof_ffi_Profile *profile;
127
+ TypedData_Get_Struct(recorder_instance, ddprof_ffi_Profile, &stack_recorder_typed_data, profile);
128
+
129
+ ddprof_ffi_Profile_add(profile, sample);
130
+ }
131
+
132
+ static void *call_serialize_without_gvl(void *call_args) {
133
+ struct call_serialize_without_gvl_arguments *args = (struct call_serialize_without_gvl_arguments *) call_args;
134
+
135
+ args->result = ddprof_ffi_Profile_serialize(args->profile);
136
+ args->serialize_ran = true;
137
+
138
+ return NULL; // Unused
139
+ }
@@ -0,0 +1,28 @@
1
+ #pragma once
2
+
3
+ #include <ddprof/ffi.h>
4
+
5
+ // Note: Please DO NOT use `VALUE_STRING` anywhere else, instead use `DDPROF_FFI_CHARSLICE_C`.
6
+ // `VALUE_STRING` is only needed because older versions of gcc (4.9.2, used in our Ruby 2.1 and 2.2 CI test images)
7
+ // tripped when compiling `enabled_value_types` using `-std=gnu99` due to the extra cast that is included in
8
+ // `DDPROF_FFI_CHARSLICE_C` with the following error:
9
+ //
10
+ // ```
11
+ // compiling ../../../../ext/ddtrace_profiling_native_extension/stack_recorder.c
12
+ // ../../../../ext/ddtrace_profiling_native_extension/stack_recorder.c:23:1: error: initializer element is not constant
13
+ // static const ddprof_ffi_ValueType enabled_value_types[] = {CPU_TIME_VALUE, CPU_SAMPLES_VALUE, WALL_TIME_VALUE};
14
+ // ^
15
+ // ```
16
+ #define VALUE_STRING(string) {.ptr = "" string, .len = sizeof(string) - 1}
17
+
18
+ #define CPU_TIME_VALUE {.type_ = VALUE_STRING("cpu-time"), .unit = VALUE_STRING("nanoseconds")}
19
+ #define CPU_SAMPLES_VALUE {.type_ = VALUE_STRING("cpu-samples"), .unit = VALUE_STRING("count")}
20
+ #define WALL_TIME_VALUE {.type_ = VALUE_STRING("wall-time"), .unit = VALUE_STRING("nanoseconds")}
21
+ #define ALLOC_SAMPLES_VALUE {.type_ = VALUE_STRING("alloc-samples"), .unit = VALUE_STRING("count")}
22
+ #define ALLOC_SPACE_VALUE {.type_ = VALUE_STRING("alloc-space"), .unit = VALUE_STRING("bytes")}
23
+ #define HEAP_SPACE_VALUE {.type_ = VALUE_STRING("heap-space"), .unit = VALUE_STRING("bytes")}
24
+
25
+ static const ddprof_ffi_ValueType enabled_value_types[] = {CPU_TIME_VALUE, CPU_SAMPLES_VALUE, WALL_TIME_VALUE};
26
+ #define ENABLED_VALUE_TYPES_COUNT (sizeof(enabled_value_types) / sizeof(ddprof_ffi_ValueType))
27
+
28
+ void record_sample(VALUE recorder_instance, ddprof_ffi_Sample sample);
@@ -4,13 +4,13 @@ if %w[1 true].include?((ENV['DD_APPSEC_ENABLED'] || '').downcase)
4
4
  begin
5
5
  require 'datadog/appsec'
6
6
  rescue StandardError => e
7
- puts "AppSec failed to load. No security check will be performed. error: #{e.message}"
7
+ puts "AppSec failed to load. No security check will be performed. error: #{e.class.name} #{e.message}"
8
8
  end
9
9
 
10
10
  begin
11
11
  require 'datadog/appsec/contrib/auto_instrument'
12
12
  Datadog::AppSec::Contrib::AutoInstrument.patch_all
13
13
  rescue StandardError => e
14
- puts "AppSec failed to instrument. No security check will be performed. error: #{e.message}"
14
+ puts "AppSec failed to instrument. No security check will be performed. error: #{e.class.name} #{e.message}"
15
15
  end
16
16
  end
@@ -5,6 +5,7 @@ module Datadog
5
5
  module Configuration
6
6
  # Configuration settings, acting as an integration registry
7
7
  # TODO: as with Configuration, this is a trivial implementation
8
+ # rubocop:disable Metrics/ClassLength
8
9
  class Settings
9
10
  class << self
10
11
  def boolean
@@ -84,12 +85,19 @@ module Datadog
84
85
  # rubocop:enable Metrics/MethodLength
85
86
  end
86
87
 
88
+ # rubocop:disable Layout/LineLength
89
+ DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'.freeze
90
+ DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'.freeze
91
+ # rubocop:enable Layout/LineLength
92
+
87
93
  DEFAULTS = {
88
94
  enabled: false,
89
95
  ruleset: :recommended,
90
96
  waf_timeout: 5_000, # us
91
97
  waf_debug: false,
92
98
  trace_rate_limit: 100, # traces/s
99
+ obfuscator_key_regex: DEFAULT_OBFUSCATOR_KEY_REGEX,
100
+ obfuscator_value_regex: DEFAULT_OBFUSCATOR_VALUE_REGEX,
93
101
  }.freeze
94
102
 
95
103
  ENVS = {
@@ -98,6 +106,8 @@ module Datadog
98
106
  'DD_APPSEC_WAF_TIMEOUT' => [:waf_timeout, Settings.duration(:us)],
99
107
  'DD_APPSEC_WAF_DEBUG' => [:waf_debug, Settings.boolean],
100
108
  'DD_APPSEC_TRACE_RATE_LIMIT' => [:trace_rate_limit, Settings.integer],
109
+ 'DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP' => [:obfuscator_key_regex, Settings.string],
110
+ 'DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP' => [:obfuscator_value_regex, Settings.string],
101
111
  }.freeze
102
112
 
103
113
  Integration = Struct.new(:integration, :options)
@@ -131,6 +141,14 @@ module Datadog
131
141
  @options[:trace_rate_limit]
132
142
  end
133
143
 
144
+ def obfuscator_key_regex
145
+ @options[:obfuscator_key_regex]
146
+ end
147
+
148
+ def obfuscator_value_regex
149
+ @options[:obfuscator_value_regex]
150
+ end
151
+
134
152
  def [](integration_name)
135
153
  integration = Datadog::AppSec::Contrib::Integration.registry[integration_name]
136
154
 
@@ -170,6 +188,7 @@ module Datadog
170
188
  initialize
171
189
  end
172
190
  end
191
+ # rubocop:enable Metrics/ClassLength
173
192
  end
174
193
  end
175
194
  end
@@ -47,6 +47,14 @@ module Datadog
47
47
  options[:trace_rate_limit] = value
48
48
  end
49
49
 
50
+ def obfuscator_key_regex=(value)
51
+ options[:obfuscator_key_regex] = value
52
+ end
53
+
54
+ def obfuscator_value_regex=(value)
55
+ options[:obfuscator_value_regex] = value
56
+ end
57
+
50
58
  def [](key)
51
59
  found = @instruments.find { |e| e.name == key }
52
60
 
@@ -3,6 +3,7 @@
3
3
  require 'datadog/appsec/instrumentation/gateway'
4
4
  require 'datadog/appsec/reactive/operation'
5
5
  require 'datadog/appsec/contrib/rack/reactive/request'
6
+ require 'datadog/appsec/contrib/rack/reactive/request_body'
6
7
  require 'datadog/appsec/contrib/rack/reactive/response'
7
8
  require 'datadog/appsec/event'
8
9
 
@@ -12,11 +13,10 @@ module Datadog
12
13
  module Rack
13
14
  module Gateway
14
15
  # Watcher for Rack gateway events
16
+ # rubocop:disable Metrics/ModuleLength
15
17
  module Watcher
16
18
  # rubocop:disable Metrics/AbcSize
17
- # rubocop:disable Metrics/CyclomaticComplexity
18
19
  # rubocop:disable Metrics/MethodLength
19
- # rubocop:disable Metrics/PerceivedComplexity
20
20
  def self.watch
21
21
  Instrumentation.gateway.watch('rack.request') do |stack, request|
22
22
  block = false
@@ -24,18 +24,8 @@ module Datadog
24
24
  waf_context = request.env['datadog.waf.context']
25
25
 
26
26
  AppSec::Reactive::Operation.new('rack.request') do |op|
27
- # TODO: factor out
28
- if defined?(Datadog::Tracing) && Datadog::Tracing.respond_to?(:active_span)
29
- active_trace = Datadog::Tracing.active_trace
30
- active_span = Datadog::Tracing.active_span
31
-
32
- Datadog.logger.debug { "active span: #{active_span.span_id}" } if active_span
33
-
34
- if active_span
35
- active_span.set_tag('_dd.appsec.enabled', 1)
36
- active_span.set_tag('_dd.runtime_family', 'ruby')
37
- end
38
- end
27
+ trace = active_trace
28
+ span = active_span
39
29
 
40
30
  Rack::Reactive::Request.subscribe(op, waf_context) do |action, result, _block|
41
31
  record = [:block, :monitor].include?(action)
@@ -43,11 +33,13 @@ module Datadog
43
33
  # TODO: should this hash be an Event instance instead?
44
34
  event = {
45
35
  waf_result: result,
46
- trace: active_trace,
47
- span: active_span,
36
+ trace: trace,
37
+ span: span,
48
38
  request: request,
49
39
  action: action
50
40
  }
41
+
42
+ waf_context.events << event
51
43
  end
52
44
  end
53
45
 
@@ -72,18 +64,8 @@ module Datadog
72
64
  waf_context = response.instance_eval { @waf_context }
73
65
 
74
66
  AppSec::Reactive::Operation.new('rack.response') do |op|
75
- # TODO: factor out
76
- if defined?(Datadog::Tracing) && Datadog::Tracing.respond_to?(:active_span)
77
- active_trace = Datadog::Tracing.active_trace
78
- active_span = Datadog::Tracing.active_span
79
-
80
- Datadog.logger.debug { "active span: #{active_span.span_id}" } if active_span
81
-
82
- if active_span
83
- active_span.set_tag('_dd.appsec.enabled', 1)
84
- active_span.set_tag('_dd.runtime_family', 'ruby')
85
- end
86
- end
67
+ trace = active_trace
68
+ span = active_span
87
69
 
88
70
  Rack::Reactive::Response.subscribe(op, waf_context) do |action, result, _block|
89
71
  record = [:block, :monitor].include?(action)
@@ -91,11 +73,13 @@ module Datadog
91
73
  # TODO: should this hash be an Event instance instead?
92
74
  event = {
93
75
  waf_result: result,
94
- trace: active_trace,
95
- span: active_span,
76
+ trace: trace,
77
+ span: span,
96
78
  response: response,
97
79
  action: action
98
80
  }
81
+
82
+ waf_context.events << event
99
83
  end
100
84
  end
101
85
 
@@ -113,12 +97,71 @@ module Datadog
113
97
 
114
98
  [ret, res]
115
99
  end
100
+
101
+ Instrumentation.gateway.watch('rack.request.body') do |stack, request|
102
+ block = false
103
+ event = nil
104
+ waf_context = request.env['datadog.waf.context']
105
+
106
+ AppSec::Reactive::Operation.new('rack.request.body') do |op|
107
+ trace = active_trace
108
+ span = active_span
109
+
110
+ Rack::Reactive::RequestBody.subscribe(op, waf_context) do |action, result, _block|
111
+ record = [:block, :monitor].include?(action)
112
+ if record
113
+ # TODO: should this hash be an Event instance instead?
114
+ event = {
115
+ waf_result: result,
116
+ trace: trace,
117
+ span: span,
118
+ request: request,
119
+ action: action
120
+ }
121
+
122
+ waf_context.events << event
123
+ end
124
+ end
125
+
126
+ _action, _result, block = Rack::Reactive::RequestBody.publish(op, request)
127
+ end
128
+
129
+ next [nil, [[:block, event]]] if block
130
+
131
+ ret, res = stack.call(request)
132
+
133
+ if event
134
+ res ||= []
135
+ res << [:monitor, event]
136
+ end
137
+
138
+ [ret, res]
139
+ end
116
140
  end
117
- # rubocop:enable Metrics/AbcSize
118
- # rubocop:enable Metrics/CyclomaticComplexity
119
141
  # rubocop:enable Metrics/MethodLength
120
- # rubocop:enable Metrics/PerceivedComplexity
142
+ # rubocop:enable Metrics/AbcSize
143
+
144
+ class << self
145
+ private
146
+
147
+ def active_trace
148
+ # TODO: factor out tracing availability detection
149
+
150
+ return unless defined?(Datadog::Tracing)
151
+
152
+ Datadog::Tracing.active_trace
153
+ end
154
+
155
+ def active_span
156
+ # TODO: factor out tracing availability detection
157
+
158
+ return unless defined?(Datadog::Tracing)
159
+
160
+ Datadog::Tracing.active_span
161
+ end
162
+ end
121
163
  end
164
+ # rubocop:enable Metrics/ModuleLength
122
165
  end
123
166
  end
124
167
  end
@@ -5,6 +5,7 @@ require 'datadog/appsec/contrib/integration'
5
5
  require 'datadog/appsec/contrib/rack/configuration/settings'
6
6
  require 'datadog/appsec/contrib/rack/patcher'
7
7
  require 'datadog/appsec/contrib/rack/request_middleware'
8
+ require 'datadog/appsec/contrib/rack/request_body_middleware'
8
9
 
9
10
  module Datadog
10
11
  module AppSec
@@ -1,7 +1,6 @@
1
1
  # typed: ignore
2
2
 
3
3
  require 'datadog/appsec/contrib/patcher'
4
- require 'datadog/appsec/contrib/rack/integration'
5
4
  require 'datadog/appsec/contrib/rack/gateway/watcher'
6
5
 
7
6
  module Datadog
@@ -0,0 +1,64 @@
1
+ # typed: true
2
+
3
+ require 'datadog/appsec/contrib/rack/request'
4
+
5
+ module Datadog
6
+ module AppSec
7
+ module Contrib
8
+ module Rack
9
+ module Reactive
10
+ # Dispatch data from a Rack request to the WAF context
11
+ module RequestBody
12
+ def self.publish(op, request)
13
+ catch(:block) do
14
+ # params have been parsed from the request body
15
+ op.publish('request.body', Rack::Request.form_hash(request))
16
+
17
+ nil
18
+ end
19
+ end
20
+
21
+ def self.subscribe(op, waf_context)
22
+ addresses = [
23
+ 'request.body',
24
+ ]
25
+
26
+ op.subscribe(*addresses) do |*values|
27
+ Datadog.logger.debug { "reacted to #{addresses.inspect}: #{values.inspect}" }
28
+ body = values[0]
29
+
30
+ waf_args = {
31
+ 'server.request.body' => body,
32
+ }
33
+
34
+ waf_timeout = Datadog::AppSec.settings.waf_timeout
35
+ action, result = waf_context.run(waf_args, waf_timeout)
36
+
37
+ Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
38
+
39
+ # TODO: encapsulate return array in a type
40
+ case action
41
+ when :monitor
42
+ Datadog.logger.debug { "WAF: #{result.inspect}" }
43
+ yield [action, result, false]
44
+ when :block
45
+ Datadog.logger.debug { "WAF: #{result.inspect}" }
46
+ yield [action, result, true]
47
+ throw(:block, [action, result, true])
48
+ when :good
49
+ Datadog.logger.debug { "WAF OK: #{result.inspect}" }
50
+ when :invalid_call
51
+ Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
52
+ when :invalid_rule, :invalid_flow, :no_rule
53
+ Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
54
+ else
55
+ Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
@@ -45,6 +45,12 @@ module Datadog
45
45
  def self.cookies(request)
46
46
  request.cookies
47
47
  end
48
+
49
+ def self.form_hash(request)
50
+ # usually Hash<String,String> but can be a more complex
51
+ # Hash<String,String||Array||Hash> when e.g coming from JSON
52
+ request.env['rack.request.form_hash']
53
+ end
48
54
  end
49
55
  end
50
56
  end
@@ -0,0 +1,41 @@
1
+ # typed: ignore
2
+
3
+ require 'datadog/appsec/instrumentation/gateway'
4
+ require 'datadog/appsec/assets'
5
+
6
+ module Datadog
7
+ module AppSec
8
+ module Contrib
9
+ module Rack
10
+ # Rack request body middleware for AppSec
11
+ # This should be inserted just below Rack::JSONBodyParser or
12
+ # legacy Rack::PostBodyContentTypeParser from rack-contrib
13
+ class RequestBodyMiddleware
14
+ def initialize(app, opt = {})
15
+ @app = app
16
+ end
17
+
18
+ def call(env)
19
+ context = env['datadog.waf.context']
20
+
21
+ return @app.call(env) unless context
22
+
23
+ # TODO: handle exceptions, except for @app.call
24
+
25
+ request = ::Rack::Request.new(env)
26
+
27
+ request_return, request_response = Instrumentation.gateway.push('rack.request.body', request) do
28
+ @app.call(env)
29
+ end
30
+
31
+ if request_response && request_response.any? { |action, _event| action == :block }
32
+ request_return = [403, { 'Content-Type' => 'text/html' }, [Datadog::AppSec::Assets.blocked]]
33
+ end
34
+
35
+ request_return
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -1,5 +1,7 @@
1
1
  # typed: ignore
2
2
 
3
+ require 'json'
4
+
3
5
  require 'datadog/appsec/instrumentation/gateway'
4
6
  require 'datadog/appsec/processor'
5
7
  require 'datadog/appsec/assets'
@@ -14,6 +16,7 @@ module Datadog
14
16
  def initialize(app, opt = {})
15
17
  @app = app
16
18
 
19
+ @oneshot_tags_sent = false
17
20
  @processor = Datadog::AppSec::Processor.new
18
21
  end
19
22
 
@@ -27,6 +30,8 @@ module Datadog
27
30
  env['datadog.waf.context'] = context
28
31
  request = ::Rack::Request.new(env)
29
32
 
33
+ add_appsec_tags
34
+
30
35
  request_return, request_response = Instrumentation.gateway.push('rack.request', request) do
31
36
  @app.call(env)
32
37
  end
@@ -40,15 +45,65 @@ module Datadog
40
45
  @waf_context = context
41
46
  end
42
47
 
43
- _, response_response = Instrumentation.gateway.push('rack.response', response)
48
+ _response_return, _response_response = Instrumentation.gateway.push('rack.response', response)
44
49
 
45
- request_response.each { |_, e| e.merge!(response: response) } if request_response
46
- response_response.each { |_, e| e.merge!(request: request) } if response_response
47
- both_response = (request_response || []) + (response_response || [])
50
+ context.events.each do |e|
51
+ e[:response] ||= response
52
+ e[:request] ||= request
53
+ end
48
54
 
49
- AppSec::Event.record(*both_response.map { |_action, event| event }) if both_response.any?
55
+ AppSec::Event.record(*context.events)
50
56
 
51
57
  request_return
58
+ ensure
59
+ add_waf_runtime_tags(context) if context
60
+ end
61
+
62
+ private
63
+
64
+ def active_trace
65
+ # TODO: factor out tracing availability detection
66
+
67
+ return unless defined?(Datadog::Tracing)
68
+
69
+ Datadog::Tracing.active_trace
70
+ end
71
+
72
+ def add_appsec_tags
73
+ return unless active_trace
74
+
75
+ active_trace.set_tag('_dd.appsec.enabled', 1)
76
+ active_trace.set_tag('_dd.runtime_family', 'ruby')
77
+ active_trace.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
78
+
79
+ if @processor.ruleset_info
80
+ active_trace.set_tag('_dd.appsec.event_rules.version', @processor.ruleset_info[:version])
81
+
82
+ unless @oneshot_tags_sent
83
+ # Small race condition, but it's inoccuous: worst case the tags
84
+ # are sent a couple of times more than expected
85
+ @oneshot_tags_sent = true
86
+
87
+ active_trace.set_tag('_dd.appsec.event_rules.loaded', @processor.ruleset_info[:loaded].to_f)
88
+ active_trace.set_tag('_dd.appsec.event_rules.error_count', @processor.ruleset_info[:failed].to_f)
89
+ active_trace.set_tag('_dd.appsec.event_rules.errors', JSON.dump(@processor.ruleset_info[:errors]))
90
+ active_trace.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(@processor.addresses))
91
+
92
+ # Ensure these tags reach the backend
93
+ active_trace.keep!
94
+ end
95
+ end
96
+ end
97
+
98
+ def add_waf_runtime_tags(context)
99
+ return unless active_trace
100
+ return unless context
101
+
102
+ active_trace.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
103
+
104
+ # these tags expect time in us
105
+ active_trace.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
106
+ active_trace.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
52
107
  end
53
108
  end
54
109
  end