ddtrace 1.0.0 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (122) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -16
  3. data/CHANGELOG.md +31 -2
  4. data/LICENSE-3rdparty.csv +3 -2
  5. data/README.md +2 -2
  6. data/ddtrace.gemspec +12 -3
  7. data/docs/GettingStarted.md +19 -2
  8. data/docs/ProfilingDevelopment.md +8 -8
  9. data/docs/UpgradeGuide.md +3 -3
  10. data/ext/ddtrace_profiling_loader/ddtrace_profiling_loader.c +118 -0
  11. data/ext/ddtrace_profiling_loader/extconf.rb +53 -0
  12. data/ext/ddtrace_profiling_native_extension/NativeExtensionDesign.md +31 -5
  13. data/ext/ddtrace_profiling_native_extension/clock_id_from_pthread.c +0 -8
  14. data/ext/ddtrace_profiling_native_extension/collectors_stack.c +278 -0
  15. data/ext/ddtrace_profiling_native_extension/extconf.rb +70 -100
  16. data/ext/ddtrace_profiling_native_extension/libddprof_helpers.h +13 -0
  17. data/ext/ddtrace_profiling_native_extension/native_extension_helpers.rb +186 -0
  18. data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c +579 -7
  19. data/ext/ddtrace_profiling_native_extension/private_vm_api_access.h +30 -0
  20. data/ext/ddtrace_profiling_native_extension/profiling.c +7 -0
  21. data/ext/ddtrace_profiling_native_extension/stack_recorder.c +139 -0
  22. data/ext/ddtrace_profiling_native_extension/stack_recorder.h +28 -0
  23. data/lib/datadog/appsec/autoload.rb +2 -2
  24. data/lib/datadog/appsec/configuration/settings.rb +19 -0
  25. data/lib/datadog/appsec/configuration.rb +8 -0
  26. data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +76 -33
  27. data/lib/datadog/appsec/contrib/rack/integration.rb +1 -0
  28. data/lib/datadog/appsec/contrib/rack/patcher.rb +0 -1
  29. data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb +64 -0
  30. data/lib/datadog/appsec/contrib/rack/request.rb +6 -0
  31. data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb +41 -0
  32. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +60 -5
  33. data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +81 -0
  34. data/lib/datadog/appsec/contrib/rails/patcher.rb +34 -1
  35. data/lib/datadog/appsec/contrib/rails/reactive/action.rb +68 -0
  36. data/lib/datadog/appsec/contrib/rails/request.rb +33 -0
  37. data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +124 -0
  38. data/lib/datadog/appsec/contrib/sinatra/patcher.rb +69 -2
  39. data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb +63 -0
  40. data/lib/datadog/appsec/event.rb +33 -18
  41. data/lib/datadog/appsec/extensions.rb +0 -3
  42. data/lib/datadog/appsec/processor.rb +45 -2
  43. data/lib/datadog/appsec/rate_limiter.rb +5 -0
  44. data/lib/datadog/appsec/reactive/operation.rb +0 -1
  45. data/lib/datadog/ci/ext/environment.rb +21 -7
  46. data/lib/datadog/core/configuration/agent_settings_resolver.rb +1 -1
  47. data/lib/datadog/core/configuration/components.rb +22 -4
  48. data/lib/datadog/core/configuration/settings.rb +3 -3
  49. data/lib/datadog/core/configuration.rb +7 -5
  50. data/lib/datadog/core/environment/cgroup.rb +3 -1
  51. data/lib/datadog/core/environment/container.rb +2 -1
  52. data/lib/datadog/core/environment/variable_helpers.rb +26 -2
  53. data/lib/datadog/core/logging/ext.rb +11 -0
  54. data/lib/datadog/core/metrics/client.rb +15 -5
  55. data/lib/datadog/core/runtime/metrics.rb +1 -1
  56. data/lib/datadog/core/workers/async.rb +3 -1
  57. data/lib/datadog/core/workers/runtime_metrics.rb +0 -3
  58. data/lib/datadog/core.rb +6 -0
  59. data/lib/datadog/kit/enable_core_dumps.rb +50 -0
  60. data/lib/datadog/kit/identity.rb +63 -0
  61. data/lib/datadog/kit.rb +11 -0
  62. data/lib/datadog/opentracer/tracer.rb +0 -2
  63. data/lib/datadog/profiling/collectors/old_stack.rb +298 -0
  64. data/lib/datadog/profiling/collectors/stack.rb +6 -287
  65. data/lib/datadog/profiling/encoding/profile.rb +0 -1
  66. data/lib/datadog/profiling/ext.rb +1 -1
  67. data/lib/datadog/profiling/flush.rb +1 -1
  68. data/lib/datadog/profiling/load_native_extension.rb +22 -0
  69. data/lib/datadog/profiling/recorder.rb +1 -1
  70. data/lib/datadog/profiling/scheduler.rb +1 -1
  71. data/lib/datadog/profiling/stack_recorder.rb +33 -0
  72. data/lib/datadog/profiling/tag_builder.rb +48 -0
  73. data/lib/datadog/profiling/tasks/exec.rb +2 -2
  74. data/lib/datadog/profiling/tasks/setup.rb +6 -4
  75. data/lib/datadog/profiling.rb +29 -27
  76. data/lib/datadog/tracing/buffer.rb +9 -3
  77. data/lib/datadog/tracing/contrib/action_view/patcher.rb +0 -1
  78. data/lib/datadog/tracing/contrib/active_record/configuration/resolver.rb +2 -2
  79. data/lib/datadog/tracing/contrib/active_record/utils.rb +1 -1
  80. data/lib/datadog/tracing/contrib/active_record/vendor/connection_specification.rb +1 -1
  81. data/lib/datadog/tracing/contrib/active_support/notifications/subscription.rb +4 -2
  82. data/lib/datadog/tracing/contrib/concurrent_ruby/context_composite_executor_service.rb +10 -3
  83. data/lib/datadog/tracing/contrib/dalli/patcher.rb +0 -1
  84. data/lib/datadog/tracing/contrib/delayed_job/patcher.rb +0 -1
  85. data/lib/datadog/tracing/contrib/elasticsearch/integration.rb +9 -3
  86. data/lib/datadog/tracing/contrib/elasticsearch/patcher.rb +38 -2
  87. data/lib/datadog/tracing/contrib/ethon/patcher.rb +0 -1
  88. data/lib/datadog/tracing/contrib/extensions.rb +0 -2
  89. data/lib/datadog/tracing/contrib/faraday/patcher.rb +0 -1
  90. data/lib/datadog/tracing/contrib/grape/patcher.rb +0 -1
  91. data/lib/datadog/tracing/contrib/graphql/patcher.rb +0 -1
  92. data/lib/datadog/tracing/contrib/grpc/patcher.rb +0 -1
  93. data/lib/datadog/tracing/contrib/kafka/patcher.rb +0 -1
  94. data/lib/datadog/tracing/contrib/lograge/instrumentation.rb +2 -1
  95. data/lib/datadog/tracing/contrib/qless/patcher.rb +0 -1
  96. data/lib/datadog/tracing/contrib/que/patcher.rb +0 -1
  97. data/lib/datadog/tracing/contrib/racecar/patcher.rb +0 -1
  98. data/lib/datadog/tracing/contrib/rails/log_injection.rb +3 -16
  99. data/lib/datadog/tracing/contrib/rake/instrumentation.rb +2 -2
  100. data/lib/datadog/tracing/contrib/rake/patcher.rb +0 -1
  101. data/lib/datadog/tracing/contrib/redis/patcher.rb +0 -1
  102. data/lib/datadog/tracing/contrib/resque/patcher.rb +0 -1
  103. data/lib/datadog/tracing/contrib/rest_client/patcher.rb +0 -1
  104. data/lib/datadog/tracing/contrib/semantic_logger/instrumentation.rb +2 -1
  105. data/lib/datadog/tracing/contrib/sidekiq/configuration/settings.rb +1 -0
  106. data/lib/datadog/tracing/contrib/sidekiq/server_tracer.rb +20 -1
  107. data/lib/datadog/tracing/contrib/sinatra/framework.rb +11 -0
  108. data/lib/datadog/tracing/contrib/sinatra/patcher.rb +0 -1
  109. data/lib/datadog/tracing/contrib/sneakers/patcher.rb +0 -1
  110. data/lib/datadog/tracing/contrib/sucker_punch/patcher.rb +0 -1
  111. data/lib/datadog/tracing/event.rb +2 -1
  112. data/lib/datadog/tracing/sampling/priority_sampler.rb +4 -5
  113. data/lib/datadog/tracing/sampling/rule.rb +12 -6
  114. data/lib/datadog/tracing/sampling/rule_sampler.rb +3 -5
  115. data/lib/datadog/tracing/span_operation.rb +2 -3
  116. data/lib/datadog/tracing/trace_operation.rb +0 -1
  117. data/lib/ddtrace/transport/http/client.rb +2 -1
  118. data/lib/ddtrace/transport/http/response.rb +34 -4
  119. data/lib/ddtrace/transport/io/client.rb +3 -1
  120. data/lib/ddtrace/version.rb +1 -1
  121. data/lib/ddtrace.rb +1 -0
  122. metadata +43 -6
@@ -0,0 +1,139 @@
1
+ #include <ruby.h>
2
+ #include <ruby/thread.h>
3
+ #include "stack_recorder.h"
4
+ #include "libddprof_helpers.h"
5
+
6
+ // Used to wrap a ddprof_ffi_Profile in a Ruby object and expose Ruby-level serialization APIs
7
+ // This file implements the native bits of the Datadog::Profiling::StackRecorder class
8
+
9
+ static VALUE ok_symbol = Qnil; // :ok in Ruby
10
+ static VALUE error_symbol = Qnil; // :error in Ruby
11
+
12
+ static ID ruby_time_from_id; // id of :ruby_time_from in Ruby
13
+
14
+ static VALUE stack_recorder_class = Qnil;
15
+
16
+ struct call_serialize_without_gvl_arguments {
17
+ ddprof_ffi_Profile *profile;
18
+ ddprof_ffi_SerializeResult result;
19
+ bool serialize_ran;
20
+ };
21
+
22
+ static VALUE _native_new(VALUE klass);
23
+ static void stack_recorder_typed_data_free(void *data);
24
+ static VALUE _native_serialize(VALUE self, VALUE recorder_instance);
25
+ static VALUE ruby_time_from(ddprof_ffi_Timespec ddprof_time);
26
+ static void *call_serialize_without_gvl(void *call_args);
27
+
28
+ void stack_recorder_init(VALUE profiling_module) {
29
+ stack_recorder_class = rb_define_class_under(profiling_module, "StackRecorder", rb_cObject);
30
+
31
+ // Instances of the StackRecorder class are going to be "TypedData" objects.
32
+ // "TypedData" objects are special objects in the Ruby VM that can wrap C structs.
33
+ // In our case, we're going to keep a libddprof profile reference inside our object.
34
+ //
35
+ // Because Ruby doesn't know how to initialize libddprof profiles, we MUST override the allocation function for objects
36
+ // of this class so that we can manage this part. Not overriding or disabling the allocation function is a common
37
+ // gotcha for "TypedData" objects that can very easily lead to VM crashes, see for instance
38
+ // https://bugs.ruby-lang.org/issues/18007 for a discussion around this.
39
+ rb_define_alloc_func(stack_recorder_class, _native_new);
40
+
41
+ rb_define_singleton_method(stack_recorder_class, "_native_serialize", _native_serialize, 1);
42
+
43
+ ok_symbol = ID2SYM(rb_intern_const("ok"));
44
+ error_symbol = ID2SYM(rb_intern_const("error"));
45
+ ruby_time_from_id = rb_intern_const("ruby_time_from");
46
+ }
47
+
48
+ // This structure is used to define a Ruby object that stores a pointer to a ddprof_ffi_Profile instance
49
+ // See also https://github.com/ruby/ruby/blob/master/doc/extension.rdoc for how this works
50
+ static const rb_data_type_t stack_recorder_typed_data = {
51
+ .wrap_struct_name = "Datadog::Profiling::StackRecorder",
52
+ .function = {
53
+ .dfree = stack_recorder_typed_data_free,
54
+ .dsize = NULL, // We don't track profile memory usage (although it'd be cool if we did!)
55
+ // No need to provide dmark nor dcompact because we don't directly reference Ruby VALUEs from inside this object
56
+ },
57
+ .flags = RUBY_TYPED_FREE_IMMEDIATELY
58
+ };
59
+
60
+ static VALUE _native_new(VALUE klass) {
61
+ ddprof_ffi_Slice_value_type sample_types = {.ptr = enabled_value_types, .len = ENABLED_VALUE_TYPES_COUNT};
62
+
63
+ ddprof_ffi_Profile *profile = ddprof_ffi_Profile_new(sample_types, NULL /* Period is optional */);
64
+
65
+ return TypedData_Wrap_Struct(klass, &stack_recorder_typed_data, profile);
66
+ }
67
+
68
+ static void stack_recorder_typed_data_free(void *data) {
69
+ ddprof_ffi_Profile_free((ddprof_ffi_Profile *) data);
70
+ }
71
+
72
+ static VALUE _native_serialize(VALUE self, VALUE recorder_instance) {
73
+ ddprof_ffi_Profile *profile;
74
+ TypedData_Get_Struct(recorder_instance, ddprof_ffi_Profile, &stack_recorder_typed_data, profile);
75
+
76
+ // We'll release the Global VM Lock while we're calling serialize, so that the Ruby VM can continue to work while this
77
+ // is pending
78
+ struct call_serialize_without_gvl_arguments args = {.profile = profile, .serialize_ran = false};
79
+
80
+ // We use rb_thread_call_without_gvl2 for similar reasons as in http_transport.c: we don't want pending interrupts
81
+ // that cause exceptions to be raised to be processed as otherwise we can leak the serialized profile.
82
+ rb_thread_call_without_gvl2(call_serialize_without_gvl, &args, /* No interruption supported */ NULL, NULL);
83
+
84
+ // This weird corner case can happen if rb_thread_call_without_gvl2 returns immediately due to an interrupt
85
+ // without ever calling call_serialize_without_gvl. In this situation, we don't have anything to clean up, we can
86
+ // just return.
87
+ if (!args.serialize_ran) {
88
+ return rb_ary_new_from_args(2, error_symbol, rb_str_new_cstr("Interrupted before call_serialize_without_gvl ran"));
89
+ }
90
+
91
+ ddprof_ffi_SerializeResult serialized_profile = args.result;
92
+
93
+ if (serialized_profile.tag == DDPROF_FFI_SERIALIZE_RESULT_ERR) {
94
+ VALUE err_details = ruby_string_from_vec_u8(serialized_profile.err);
95
+ ddprof_ffi_SerializeResult_drop(serialized_profile);
96
+ return rb_ary_new_from_args(2, error_symbol, err_details);
97
+ }
98
+
99
+ VALUE encoded_pprof = ruby_string_from_vec_u8(serialized_profile.ok.buffer);
100
+
101
+ ddprof_ffi_Timespec ddprof_start = serialized_profile.ok.start;
102
+ ddprof_ffi_Timespec ddprof_finish = serialized_profile.ok.end;
103
+
104
+ // Clean up libddprof object to avoid leaking in case ruby_time_from raises an exception
105
+ ddprof_ffi_SerializeResult_drop(serialized_profile);
106
+
107
+ VALUE start = ruby_time_from(ddprof_start);
108
+ VALUE finish = ruby_time_from(ddprof_finish);
109
+
110
+ if (!ddprof_ffi_Profile_reset(profile)) return rb_ary_new_from_args(2, error_symbol, rb_str_new_cstr("Failed to reset profile"));
111
+
112
+ return rb_ary_new_from_args(2, ok_symbol, rb_ary_new_from_args(3, start, finish, encoded_pprof));
113
+ }
114
+
115
+ static VALUE ruby_time_from(ddprof_ffi_Timespec ddprof_time) {
116
+ #ifndef NO_RB_TIME_TIMESPEC_NEW // Modern Rubies
117
+ const int utc = INT_MAX - 1; // From Ruby sources
118
+ struct timespec time = {.tv_sec = ddprof_time.seconds, .tv_nsec = ddprof_time.nanoseconds};
119
+ return rb_time_timespec_new(&time, utc);
120
+ #else // Ruby < 2.3
121
+ return rb_funcall(stack_recorder_class, ruby_time_from_id, 2, LONG2NUM(ddprof_time.seconds), UINT2NUM(ddprof_time.nanoseconds));
122
+ #endif
123
+ }
124
+
125
+ void record_sample(VALUE recorder_instance, ddprof_ffi_Sample sample) {
126
+ ddprof_ffi_Profile *profile;
127
+ TypedData_Get_Struct(recorder_instance, ddprof_ffi_Profile, &stack_recorder_typed_data, profile);
128
+
129
+ ddprof_ffi_Profile_add(profile, sample);
130
+ }
131
+
132
+ static void *call_serialize_without_gvl(void *call_args) {
133
+ struct call_serialize_without_gvl_arguments *args = (struct call_serialize_without_gvl_arguments *) call_args;
134
+
135
+ args->result = ddprof_ffi_Profile_serialize(args->profile);
136
+ args->serialize_ran = true;
137
+
138
+ return NULL; // Unused
139
+ }
@@ -0,0 +1,28 @@
1
+ #pragma once
2
+
3
+ #include <ddprof/ffi.h>
4
+
5
+ // Note: Please DO NOT use `VALUE_STRING` anywhere else, instead use `DDPROF_FFI_CHARSLICE_C`.
6
+ // `VALUE_STRING` is only needed because older versions of gcc (4.9.2, used in our Ruby 2.1 and 2.2 CI test images)
7
+ // tripped when compiling `enabled_value_types` using `-std=gnu99` due to the extra cast that is included in
8
+ // `DDPROF_FFI_CHARSLICE_C` with the following error:
9
+ //
10
+ // ```
11
+ // compiling ../../../../ext/ddtrace_profiling_native_extension/stack_recorder.c
12
+ // ../../../../ext/ddtrace_profiling_native_extension/stack_recorder.c:23:1: error: initializer element is not constant
13
+ // static const ddprof_ffi_ValueType enabled_value_types[] = {CPU_TIME_VALUE, CPU_SAMPLES_VALUE, WALL_TIME_VALUE};
14
+ // ^
15
+ // ```
16
+ #define VALUE_STRING(string) {.ptr = "" string, .len = sizeof(string) - 1}
17
+
18
+ #define CPU_TIME_VALUE {.type_ = VALUE_STRING("cpu-time"), .unit = VALUE_STRING("nanoseconds")}
19
+ #define CPU_SAMPLES_VALUE {.type_ = VALUE_STRING("cpu-samples"), .unit = VALUE_STRING("count")}
20
+ #define WALL_TIME_VALUE {.type_ = VALUE_STRING("wall-time"), .unit = VALUE_STRING("nanoseconds")}
21
+ #define ALLOC_SAMPLES_VALUE {.type_ = VALUE_STRING("alloc-samples"), .unit = VALUE_STRING("count")}
22
+ #define ALLOC_SPACE_VALUE {.type_ = VALUE_STRING("alloc-space"), .unit = VALUE_STRING("bytes")}
23
+ #define HEAP_SPACE_VALUE {.type_ = VALUE_STRING("heap-space"), .unit = VALUE_STRING("bytes")}
24
+
25
+ static const ddprof_ffi_ValueType enabled_value_types[] = {CPU_TIME_VALUE, CPU_SAMPLES_VALUE, WALL_TIME_VALUE};
26
+ #define ENABLED_VALUE_TYPES_COUNT (sizeof(enabled_value_types) / sizeof(ddprof_ffi_ValueType))
27
+
28
+ void record_sample(VALUE recorder_instance, ddprof_ffi_Sample sample);
@@ -4,13 +4,13 @@ if %w[1 true].include?((ENV['DD_APPSEC_ENABLED'] || '').downcase)
4
4
  begin
5
5
  require 'datadog/appsec'
6
6
  rescue StandardError => e
7
- puts "AppSec failed to load. No security check will be performed. error: #{e.message}"
7
+ puts "AppSec failed to load. No security check will be performed. error: #{e.class.name} #{e.message}"
8
8
  end
9
9
 
10
10
  begin
11
11
  require 'datadog/appsec/contrib/auto_instrument'
12
12
  Datadog::AppSec::Contrib::AutoInstrument.patch_all
13
13
  rescue StandardError => e
14
- puts "AppSec failed to instrument. No security check will be performed. error: #{e.message}"
14
+ puts "AppSec failed to instrument. No security check will be performed. error: #{e.class.name} #{e.message}"
15
15
  end
16
16
  end
@@ -5,6 +5,7 @@ module Datadog
5
5
  module Configuration
6
6
  # Configuration settings, acting as an integration registry
7
7
  # TODO: as with Configuration, this is a trivial implementation
8
+ # rubocop:disable Metrics/ClassLength
8
9
  class Settings
9
10
  class << self
10
11
  def boolean
@@ -84,12 +85,19 @@ module Datadog
84
85
  # rubocop:enable Metrics/MethodLength
85
86
  end
86
87
 
88
+ # rubocop:disable Layout/LineLength
89
+ DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'.freeze
90
+ DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'.freeze
91
+ # rubocop:enable Layout/LineLength
92
+
87
93
  DEFAULTS = {
88
94
  enabled: false,
89
95
  ruleset: :recommended,
90
96
  waf_timeout: 5_000, # us
91
97
  waf_debug: false,
92
98
  trace_rate_limit: 100, # traces/s
99
+ obfuscator_key_regex: DEFAULT_OBFUSCATOR_KEY_REGEX,
100
+ obfuscator_value_regex: DEFAULT_OBFUSCATOR_VALUE_REGEX,
93
101
  }.freeze
94
102
 
95
103
  ENVS = {
@@ -98,6 +106,8 @@ module Datadog
98
106
  'DD_APPSEC_WAF_TIMEOUT' => [:waf_timeout, Settings.duration(:us)],
99
107
  'DD_APPSEC_WAF_DEBUG' => [:waf_debug, Settings.boolean],
100
108
  'DD_APPSEC_TRACE_RATE_LIMIT' => [:trace_rate_limit, Settings.integer],
109
+ 'DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP' => [:obfuscator_key_regex, Settings.string],
110
+ 'DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP' => [:obfuscator_value_regex, Settings.string],
101
111
  }.freeze
102
112
 
103
113
  Integration = Struct.new(:integration, :options)
@@ -131,6 +141,14 @@ module Datadog
131
141
  @options[:trace_rate_limit]
132
142
  end
133
143
 
144
+ def obfuscator_key_regex
145
+ @options[:obfuscator_key_regex]
146
+ end
147
+
148
+ def obfuscator_value_regex
149
+ @options[:obfuscator_value_regex]
150
+ end
151
+
134
152
  def [](integration_name)
135
153
  integration = Datadog::AppSec::Contrib::Integration.registry[integration_name]
136
154
 
@@ -170,6 +188,7 @@ module Datadog
170
188
  initialize
171
189
  end
172
190
  end
191
+ # rubocop:enable Metrics/ClassLength
173
192
  end
174
193
  end
175
194
  end
@@ -47,6 +47,14 @@ module Datadog
47
47
  options[:trace_rate_limit] = value
48
48
  end
49
49
 
50
+ def obfuscator_key_regex=(value)
51
+ options[:obfuscator_key_regex] = value
52
+ end
53
+
54
+ def obfuscator_value_regex=(value)
55
+ options[:obfuscator_value_regex] = value
56
+ end
57
+
50
58
  def [](key)
51
59
  found = @instruments.find { |e| e.name == key }
52
60
 
@@ -3,6 +3,7 @@
3
3
  require 'datadog/appsec/instrumentation/gateway'
4
4
  require 'datadog/appsec/reactive/operation'
5
5
  require 'datadog/appsec/contrib/rack/reactive/request'
6
+ require 'datadog/appsec/contrib/rack/reactive/request_body'
6
7
  require 'datadog/appsec/contrib/rack/reactive/response'
7
8
  require 'datadog/appsec/event'
8
9
 
@@ -12,11 +13,10 @@ module Datadog
12
13
  module Rack
13
14
  module Gateway
14
15
  # Watcher for Rack gateway events
16
+ # rubocop:disable Metrics/ModuleLength
15
17
  module Watcher
16
18
  # rubocop:disable Metrics/AbcSize
17
- # rubocop:disable Metrics/CyclomaticComplexity
18
19
  # rubocop:disable Metrics/MethodLength
19
- # rubocop:disable Metrics/PerceivedComplexity
20
20
  def self.watch
21
21
  Instrumentation.gateway.watch('rack.request') do |stack, request|
22
22
  block = false
@@ -24,18 +24,8 @@ module Datadog
24
24
  waf_context = request.env['datadog.waf.context']
25
25
 
26
26
  AppSec::Reactive::Operation.new('rack.request') do |op|
27
- # TODO: factor out
28
- if defined?(Datadog::Tracing) && Datadog::Tracing.respond_to?(:active_span)
29
- active_trace = Datadog::Tracing.active_trace
30
- active_span = Datadog::Tracing.active_span
31
-
32
- Datadog.logger.debug { "active span: #{active_span.span_id}" } if active_span
33
-
34
- if active_span
35
- active_span.set_tag('_dd.appsec.enabled', 1)
36
- active_span.set_tag('_dd.runtime_family', 'ruby')
37
- end
38
- end
27
+ trace = active_trace
28
+ span = active_span
39
29
 
40
30
  Rack::Reactive::Request.subscribe(op, waf_context) do |action, result, _block|
41
31
  record = [:block, :monitor].include?(action)
@@ -43,11 +33,13 @@ module Datadog
43
33
  # TODO: should this hash be an Event instance instead?
44
34
  event = {
45
35
  waf_result: result,
46
- trace: active_trace,
47
- span: active_span,
36
+ trace: trace,
37
+ span: span,
48
38
  request: request,
49
39
  action: action
50
40
  }
41
+
42
+ waf_context.events << event
51
43
  end
52
44
  end
53
45
 
@@ -72,18 +64,8 @@ module Datadog
72
64
  waf_context = response.instance_eval { @waf_context }
73
65
 
74
66
  AppSec::Reactive::Operation.new('rack.response') do |op|
75
- # TODO: factor out
76
- if defined?(Datadog::Tracing) && Datadog::Tracing.respond_to?(:active_span)
77
- active_trace = Datadog::Tracing.active_trace
78
- active_span = Datadog::Tracing.active_span
79
-
80
- Datadog.logger.debug { "active span: #{active_span.span_id}" } if active_span
81
-
82
- if active_span
83
- active_span.set_tag('_dd.appsec.enabled', 1)
84
- active_span.set_tag('_dd.runtime_family', 'ruby')
85
- end
86
- end
67
+ trace = active_trace
68
+ span = active_span
87
69
 
88
70
  Rack::Reactive::Response.subscribe(op, waf_context) do |action, result, _block|
89
71
  record = [:block, :monitor].include?(action)
@@ -91,11 +73,13 @@ module Datadog
91
73
  # TODO: should this hash be an Event instance instead?
92
74
  event = {
93
75
  waf_result: result,
94
- trace: active_trace,
95
- span: active_span,
76
+ trace: trace,
77
+ span: span,
96
78
  response: response,
97
79
  action: action
98
80
  }
81
+
82
+ waf_context.events << event
99
83
  end
100
84
  end
101
85
 
@@ -113,12 +97,71 @@ module Datadog
113
97
 
114
98
  [ret, res]
115
99
  end
100
+
101
+ Instrumentation.gateway.watch('rack.request.body') do |stack, request|
102
+ block = false
103
+ event = nil
104
+ waf_context = request.env['datadog.waf.context']
105
+
106
+ AppSec::Reactive::Operation.new('rack.request.body') do |op|
107
+ trace = active_trace
108
+ span = active_span
109
+
110
+ Rack::Reactive::RequestBody.subscribe(op, waf_context) do |action, result, _block|
111
+ record = [:block, :monitor].include?(action)
112
+ if record
113
+ # TODO: should this hash be an Event instance instead?
114
+ event = {
115
+ waf_result: result,
116
+ trace: trace,
117
+ span: span,
118
+ request: request,
119
+ action: action
120
+ }
121
+
122
+ waf_context.events << event
123
+ end
124
+ end
125
+
126
+ _action, _result, block = Rack::Reactive::RequestBody.publish(op, request)
127
+ end
128
+
129
+ next [nil, [[:block, event]]] if block
130
+
131
+ ret, res = stack.call(request)
132
+
133
+ if event
134
+ res ||= []
135
+ res << [:monitor, event]
136
+ end
137
+
138
+ [ret, res]
139
+ end
116
140
  end
117
- # rubocop:enable Metrics/AbcSize
118
- # rubocop:enable Metrics/CyclomaticComplexity
119
141
  # rubocop:enable Metrics/MethodLength
120
- # rubocop:enable Metrics/PerceivedComplexity
142
+ # rubocop:enable Metrics/AbcSize
143
+
144
+ class << self
145
+ private
146
+
147
+ def active_trace
148
+ # TODO: factor out tracing availability detection
149
+
150
+ return unless defined?(Datadog::Tracing)
151
+
152
+ Datadog::Tracing.active_trace
153
+ end
154
+
155
+ def active_span
156
+ # TODO: factor out tracing availability detection
157
+
158
+ return unless defined?(Datadog::Tracing)
159
+
160
+ Datadog::Tracing.active_span
161
+ end
162
+ end
121
163
  end
164
+ # rubocop:enable Metrics/ModuleLength
122
165
  end
123
166
  end
124
167
  end
@@ -5,6 +5,7 @@ require 'datadog/appsec/contrib/integration'
5
5
  require 'datadog/appsec/contrib/rack/configuration/settings'
6
6
  require 'datadog/appsec/contrib/rack/patcher'
7
7
  require 'datadog/appsec/contrib/rack/request_middleware'
8
+ require 'datadog/appsec/contrib/rack/request_body_middleware'
8
9
 
9
10
  module Datadog
10
11
  module AppSec
@@ -1,7 +1,6 @@
1
1
  # typed: ignore
2
2
 
3
3
  require 'datadog/appsec/contrib/patcher'
4
- require 'datadog/appsec/contrib/rack/integration'
5
4
  require 'datadog/appsec/contrib/rack/gateway/watcher'
6
5
 
7
6
  module Datadog
@@ -0,0 +1,64 @@
1
+ # typed: true
2
+
3
+ require 'datadog/appsec/contrib/rack/request'
4
+
5
+ module Datadog
6
+ module AppSec
7
+ module Contrib
8
+ module Rack
9
+ module Reactive
10
+ # Dispatch data from a Rack request to the WAF context
11
+ module RequestBody
12
+ def self.publish(op, request)
13
+ catch(:block) do
14
+ # params have been parsed from the request body
15
+ op.publish('request.body', Rack::Request.form_hash(request))
16
+
17
+ nil
18
+ end
19
+ end
20
+
21
+ def self.subscribe(op, waf_context)
22
+ addresses = [
23
+ 'request.body',
24
+ ]
25
+
26
+ op.subscribe(*addresses) do |*values|
27
+ Datadog.logger.debug { "reacted to #{addresses.inspect}: #{values.inspect}" }
28
+ body = values[0]
29
+
30
+ waf_args = {
31
+ 'server.request.body' => body,
32
+ }
33
+
34
+ waf_timeout = Datadog::AppSec.settings.waf_timeout
35
+ action, result = waf_context.run(waf_args, waf_timeout)
36
+
37
+ Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
38
+
39
+ # TODO: encapsulate return array in a type
40
+ case action
41
+ when :monitor
42
+ Datadog.logger.debug { "WAF: #{result.inspect}" }
43
+ yield [action, result, false]
44
+ when :block
45
+ Datadog.logger.debug { "WAF: #{result.inspect}" }
46
+ yield [action, result, true]
47
+ throw(:block, [action, result, true])
48
+ when :good
49
+ Datadog.logger.debug { "WAF OK: #{result.inspect}" }
50
+ when :invalid_call
51
+ Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
52
+ when :invalid_rule, :invalid_flow, :no_rule
53
+ Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
54
+ else
55
+ Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end
@@ -45,6 +45,12 @@ module Datadog
45
45
  def self.cookies(request)
46
46
  request.cookies
47
47
  end
48
+
49
+ def self.form_hash(request)
50
+ # usually Hash<String,String> but can be a more complex
51
+ # Hash<String,String||Array||Hash> when e.g coming from JSON
52
+ request.env['rack.request.form_hash']
53
+ end
48
54
  end
49
55
  end
50
56
  end
@@ -0,0 +1,41 @@
1
+ # typed: ignore
2
+
3
+ require 'datadog/appsec/instrumentation/gateway'
4
+ require 'datadog/appsec/assets'
5
+
6
+ module Datadog
7
+ module AppSec
8
+ module Contrib
9
+ module Rack
10
+ # Rack request body middleware for AppSec
11
+ # This should be inserted just below Rack::JSONBodyParser or
12
+ # legacy Rack::PostBodyContentTypeParser from rack-contrib
13
+ class RequestBodyMiddleware
14
+ def initialize(app, opt = {})
15
+ @app = app
16
+ end
17
+
18
+ def call(env)
19
+ context = env['datadog.waf.context']
20
+
21
+ return @app.call(env) unless context
22
+
23
+ # TODO: handle exceptions, except for @app.call
24
+
25
+ request = ::Rack::Request.new(env)
26
+
27
+ request_return, request_response = Instrumentation.gateway.push('rack.request.body', request) do
28
+ @app.call(env)
29
+ end
30
+
31
+ if request_response && request_response.any? { |action, _event| action == :block }
32
+ request_return = [403, { 'Content-Type' => 'text/html' }, [Datadog::AppSec::Assets.blocked]]
33
+ end
34
+
35
+ request_return
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -1,5 +1,7 @@
1
1
  # typed: ignore
2
2
 
3
+ require 'json'
4
+
3
5
  require 'datadog/appsec/instrumentation/gateway'
4
6
  require 'datadog/appsec/processor'
5
7
  require 'datadog/appsec/assets'
@@ -14,6 +16,7 @@ module Datadog
14
16
  def initialize(app, opt = {})
15
17
  @app = app
16
18
 
19
+ @oneshot_tags_sent = false
17
20
  @processor = Datadog::AppSec::Processor.new
18
21
  end
19
22
 
@@ -27,6 +30,8 @@ module Datadog
27
30
  env['datadog.waf.context'] = context
28
31
  request = ::Rack::Request.new(env)
29
32
 
33
+ add_appsec_tags
34
+
30
35
  request_return, request_response = Instrumentation.gateway.push('rack.request', request) do
31
36
  @app.call(env)
32
37
  end
@@ -40,15 +45,65 @@ module Datadog
40
45
  @waf_context = context
41
46
  end
42
47
 
43
- _, response_response = Instrumentation.gateway.push('rack.response', response)
48
+ _response_return, _response_response = Instrumentation.gateway.push('rack.response', response)
44
49
 
45
- request_response.each { |_, e| e.merge!(response: response) } if request_response
46
- response_response.each { |_, e| e.merge!(request: request) } if response_response
47
- both_response = (request_response || []) + (response_response || [])
50
+ context.events.each do |e|
51
+ e[:response] ||= response
52
+ e[:request] ||= request
53
+ end
48
54
 
49
- AppSec::Event.record(*both_response.map { |_action, event| event }) if both_response.any?
55
+ AppSec::Event.record(*context.events)
50
56
 
51
57
  request_return
58
+ ensure
59
+ add_waf_runtime_tags(context) if context
60
+ end
61
+
62
+ private
63
+
64
+ def active_trace
65
+ # TODO: factor out tracing availability detection
66
+
67
+ return unless defined?(Datadog::Tracing)
68
+
69
+ Datadog::Tracing.active_trace
70
+ end
71
+
72
+ def add_appsec_tags
73
+ return unless active_trace
74
+
75
+ active_trace.set_tag('_dd.appsec.enabled', 1)
76
+ active_trace.set_tag('_dd.runtime_family', 'ruby')
77
+ active_trace.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
78
+
79
+ if @processor.ruleset_info
80
+ active_trace.set_tag('_dd.appsec.event_rules.version', @processor.ruleset_info[:version])
81
+
82
+ unless @oneshot_tags_sent
83
+ # Small race condition, but it's inoccuous: worst case the tags
84
+ # are sent a couple of times more than expected
85
+ @oneshot_tags_sent = true
86
+
87
+ active_trace.set_tag('_dd.appsec.event_rules.loaded', @processor.ruleset_info[:loaded].to_f)
88
+ active_trace.set_tag('_dd.appsec.event_rules.error_count', @processor.ruleset_info[:failed].to_f)
89
+ active_trace.set_tag('_dd.appsec.event_rules.errors', JSON.dump(@processor.ruleset_info[:errors]))
90
+ active_trace.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(@processor.addresses))
91
+
92
+ # Ensure these tags reach the backend
93
+ active_trace.keep!
94
+ end
95
+ end
96
+ end
97
+
98
+ def add_waf_runtime_tags(context)
99
+ return unless active_trace
100
+ return unless context
101
+
102
+ active_trace.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
103
+
104
+ # these tags expect time in us
105
+ active_trace.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
106
+ active_trace.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
52
107
  end
53
108
  end
54
109
  end