ddtrace 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +4 -16
- data/CHANGELOG.md +31 -2
- data/LICENSE-3rdparty.csv +3 -2
- data/README.md +2 -2
- data/ddtrace.gemspec +12 -3
- data/docs/GettingStarted.md +19 -2
- data/docs/ProfilingDevelopment.md +8 -8
- data/docs/UpgradeGuide.md +3 -3
- data/ext/ddtrace_profiling_loader/ddtrace_profiling_loader.c +118 -0
- data/ext/ddtrace_profiling_loader/extconf.rb +53 -0
- data/ext/ddtrace_profiling_native_extension/NativeExtensionDesign.md +31 -5
- data/ext/ddtrace_profiling_native_extension/clock_id_from_pthread.c +0 -8
- data/ext/ddtrace_profiling_native_extension/collectors_stack.c +278 -0
- data/ext/ddtrace_profiling_native_extension/extconf.rb +70 -100
- data/ext/ddtrace_profiling_native_extension/libddprof_helpers.h +13 -0
- data/ext/ddtrace_profiling_native_extension/native_extension_helpers.rb +186 -0
- data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c +579 -7
- data/ext/ddtrace_profiling_native_extension/private_vm_api_access.h +30 -0
- data/ext/ddtrace_profiling_native_extension/profiling.c +7 -0
- data/ext/ddtrace_profiling_native_extension/stack_recorder.c +139 -0
- data/ext/ddtrace_profiling_native_extension/stack_recorder.h +28 -0
- data/lib/datadog/appsec/autoload.rb +2 -2
- data/lib/datadog/appsec/configuration/settings.rb +19 -0
- data/lib/datadog/appsec/configuration.rb +8 -0
- data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +76 -33
- data/lib/datadog/appsec/contrib/rack/integration.rb +1 -0
- data/lib/datadog/appsec/contrib/rack/patcher.rb +0 -1
- data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb +64 -0
- data/lib/datadog/appsec/contrib/rack/request.rb +6 -0
- data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb +41 -0
- data/lib/datadog/appsec/contrib/rack/request_middleware.rb +60 -5
- data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +81 -0
- data/lib/datadog/appsec/contrib/rails/patcher.rb +34 -1
- data/lib/datadog/appsec/contrib/rails/reactive/action.rb +68 -0
- data/lib/datadog/appsec/contrib/rails/request.rb +33 -0
- data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +124 -0
- data/lib/datadog/appsec/contrib/sinatra/patcher.rb +69 -2
- data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb +63 -0
- data/lib/datadog/appsec/event.rb +33 -18
- data/lib/datadog/appsec/extensions.rb +0 -3
- data/lib/datadog/appsec/processor.rb +45 -2
- data/lib/datadog/appsec/rate_limiter.rb +5 -0
- data/lib/datadog/appsec/reactive/operation.rb +0 -1
- data/lib/datadog/ci/ext/environment.rb +21 -7
- data/lib/datadog/core/configuration/agent_settings_resolver.rb +1 -1
- data/lib/datadog/core/configuration/components.rb +22 -4
- data/lib/datadog/core/configuration/settings.rb +3 -3
- data/lib/datadog/core/configuration.rb +7 -5
- data/lib/datadog/core/environment/cgroup.rb +3 -1
- data/lib/datadog/core/environment/container.rb +2 -1
- data/lib/datadog/core/environment/variable_helpers.rb +26 -2
- data/lib/datadog/core/logging/ext.rb +11 -0
- data/lib/datadog/core/metrics/client.rb +15 -5
- data/lib/datadog/core/runtime/metrics.rb +1 -1
- data/lib/datadog/core/workers/async.rb +3 -1
- data/lib/datadog/core/workers/runtime_metrics.rb +0 -3
- data/lib/datadog/core.rb +6 -0
- data/lib/datadog/kit/enable_core_dumps.rb +50 -0
- data/lib/datadog/kit/identity.rb +63 -0
- data/lib/datadog/kit.rb +11 -0
- data/lib/datadog/opentracer/tracer.rb +0 -2
- data/lib/datadog/profiling/collectors/old_stack.rb +298 -0
- data/lib/datadog/profiling/collectors/stack.rb +6 -287
- data/lib/datadog/profiling/encoding/profile.rb +0 -1
- data/lib/datadog/profiling/ext.rb +1 -1
- data/lib/datadog/profiling/flush.rb +1 -1
- data/lib/datadog/profiling/load_native_extension.rb +22 -0
- data/lib/datadog/profiling/recorder.rb +1 -1
- data/lib/datadog/profiling/scheduler.rb +1 -1
- data/lib/datadog/profiling/stack_recorder.rb +33 -0
- data/lib/datadog/profiling/tag_builder.rb +48 -0
- data/lib/datadog/profiling/tasks/exec.rb +2 -2
- data/lib/datadog/profiling/tasks/setup.rb +6 -4
- data/lib/datadog/profiling.rb +29 -27
- data/lib/datadog/tracing/buffer.rb +9 -3
- data/lib/datadog/tracing/contrib/action_view/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/active_record/configuration/resolver.rb +2 -2
- data/lib/datadog/tracing/contrib/active_record/utils.rb +1 -1
- data/lib/datadog/tracing/contrib/active_record/vendor/connection_specification.rb +1 -1
- data/lib/datadog/tracing/contrib/active_support/notifications/subscription.rb +4 -2
- data/lib/datadog/tracing/contrib/concurrent_ruby/context_composite_executor_service.rb +10 -3
- data/lib/datadog/tracing/contrib/dalli/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/delayed_job/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/elasticsearch/integration.rb +9 -3
- data/lib/datadog/tracing/contrib/elasticsearch/patcher.rb +38 -2
- data/lib/datadog/tracing/contrib/ethon/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/extensions.rb +0 -2
- data/lib/datadog/tracing/contrib/faraday/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/grape/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/graphql/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/grpc/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/kafka/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/lograge/instrumentation.rb +2 -1
- data/lib/datadog/tracing/contrib/qless/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/que/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/racecar/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/rails/log_injection.rb +3 -16
- data/lib/datadog/tracing/contrib/rake/instrumentation.rb +2 -2
- data/lib/datadog/tracing/contrib/rake/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/redis/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/resque/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/rest_client/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/semantic_logger/instrumentation.rb +2 -1
- data/lib/datadog/tracing/contrib/sidekiq/configuration/settings.rb +1 -0
- data/lib/datadog/tracing/contrib/sidekiq/server_tracer.rb +20 -1
- data/lib/datadog/tracing/contrib/sinatra/framework.rb +11 -0
- data/lib/datadog/tracing/contrib/sinatra/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/sneakers/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/sucker_punch/patcher.rb +0 -1
- data/lib/datadog/tracing/event.rb +2 -1
- data/lib/datadog/tracing/sampling/priority_sampler.rb +4 -5
- data/lib/datadog/tracing/sampling/rule.rb +12 -6
- data/lib/datadog/tracing/sampling/rule_sampler.rb +3 -5
- data/lib/datadog/tracing/span_operation.rb +2 -3
- data/lib/datadog/tracing/trace_operation.rb +0 -1
- data/lib/ddtrace/transport/http/client.rb +2 -1
- data/lib/ddtrace/transport/http/response.rb +34 -4
- data/lib/ddtrace/transport/io/client.rb +3 -1
- data/lib/ddtrace/version.rb +1 -1
- data/lib/ddtrace.rb +1 -0
- metadata +43 -6
@@ -0,0 +1,139 @@
|
|
1
|
+
#include <ruby.h>
|
2
|
+
#include <ruby/thread.h>
|
3
|
+
#include "stack_recorder.h"
|
4
|
+
#include "libddprof_helpers.h"
|
5
|
+
|
6
|
+
// Used to wrap a ddprof_ffi_Profile in a Ruby object and expose Ruby-level serialization APIs
|
7
|
+
// This file implements the native bits of the Datadog::Profiling::StackRecorder class
|
8
|
+
|
9
|
+
static VALUE ok_symbol = Qnil; // :ok in Ruby
|
10
|
+
static VALUE error_symbol = Qnil; // :error in Ruby
|
11
|
+
|
12
|
+
static ID ruby_time_from_id; // id of :ruby_time_from in Ruby
|
13
|
+
|
14
|
+
static VALUE stack_recorder_class = Qnil;
|
15
|
+
|
16
|
+
struct call_serialize_without_gvl_arguments {
|
17
|
+
ddprof_ffi_Profile *profile;
|
18
|
+
ddprof_ffi_SerializeResult result;
|
19
|
+
bool serialize_ran;
|
20
|
+
};
|
21
|
+
|
22
|
+
static VALUE _native_new(VALUE klass);
|
23
|
+
static void stack_recorder_typed_data_free(void *data);
|
24
|
+
static VALUE _native_serialize(VALUE self, VALUE recorder_instance);
|
25
|
+
static VALUE ruby_time_from(ddprof_ffi_Timespec ddprof_time);
|
26
|
+
static void *call_serialize_without_gvl(void *call_args);
|
27
|
+
|
28
|
+
void stack_recorder_init(VALUE profiling_module) {
|
29
|
+
stack_recorder_class = rb_define_class_under(profiling_module, "StackRecorder", rb_cObject);
|
30
|
+
|
31
|
+
// Instances of the StackRecorder class are going to be "TypedData" objects.
|
32
|
+
// "TypedData" objects are special objects in the Ruby VM that can wrap C structs.
|
33
|
+
// In our case, we're going to keep a libddprof profile reference inside our object.
|
34
|
+
//
|
35
|
+
// Because Ruby doesn't know how to initialize libddprof profiles, we MUST override the allocation function for objects
|
36
|
+
// of this class so that we can manage this part. Not overriding or disabling the allocation function is a common
|
37
|
+
// gotcha for "TypedData" objects that can very easily lead to VM crashes, see for instance
|
38
|
+
// https://bugs.ruby-lang.org/issues/18007 for a discussion around this.
|
39
|
+
rb_define_alloc_func(stack_recorder_class, _native_new);
|
40
|
+
|
41
|
+
rb_define_singleton_method(stack_recorder_class, "_native_serialize", _native_serialize, 1);
|
42
|
+
|
43
|
+
ok_symbol = ID2SYM(rb_intern_const("ok"));
|
44
|
+
error_symbol = ID2SYM(rb_intern_const("error"));
|
45
|
+
ruby_time_from_id = rb_intern_const("ruby_time_from");
|
46
|
+
}
|
47
|
+
|
48
|
+
// This structure is used to define a Ruby object that stores a pointer to a ddprof_ffi_Profile instance
|
49
|
+
// See also https://github.com/ruby/ruby/blob/master/doc/extension.rdoc for how this works
|
50
|
+
static const rb_data_type_t stack_recorder_typed_data = {
|
51
|
+
.wrap_struct_name = "Datadog::Profiling::StackRecorder",
|
52
|
+
.function = {
|
53
|
+
.dfree = stack_recorder_typed_data_free,
|
54
|
+
.dsize = NULL, // We don't track profile memory usage (although it'd be cool if we did!)
|
55
|
+
// No need to provide dmark nor dcompact because we don't directly reference Ruby VALUEs from inside this object
|
56
|
+
},
|
57
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY
|
58
|
+
};
|
59
|
+
|
60
|
+
static VALUE _native_new(VALUE klass) {
|
61
|
+
ddprof_ffi_Slice_value_type sample_types = {.ptr = enabled_value_types, .len = ENABLED_VALUE_TYPES_COUNT};
|
62
|
+
|
63
|
+
ddprof_ffi_Profile *profile = ddprof_ffi_Profile_new(sample_types, NULL /* Period is optional */);
|
64
|
+
|
65
|
+
return TypedData_Wrap_Struct(klass, &stack_recorder_typed_data, profile);
|
66
|
+
}
|
67
|
+
|
68
|
+
static void stack_recorder_typed_data_free(void *data) {
|
69
|
+
ddprof_ffi_Profile_free((ddprof_ffi_Profile *) data);
|
70
|
+
}
|
71
|
+
|
72
|
+
static VALUE _native_serialize(VALUE self, VALUE recorder_instance) {
|
73
|
+
ddprof_ffi_Profile *profile;
|
74
|
+
TypedData_Get_Struct(recorder_instance, ddprof_ffi_Profile, &stack_recorder_typed_data, profile);
|
75
|
+
|
76
|
+
// We'll release the Global VM Lock while we're calling serialize, so that the Ruby VM can continue to work while this
|
77
|
+
// is pending
|
78
|
+
struct call_serialize_without_gvl_arguments args = {.profile = profile, .serialize_ran = false};
|
79
|
+
|
80
|
+
// We use rb_thread_call_without_gvl2 for similar reasons as in http_transport.c: we don't want pending interrupts
|
81
|
+
// that cause exceptions to be raised to be processed as otherwise we can leak the serialized profile.
|
82
|
+
rb_thread_call_without_gvl2(call_serialize_without_gvl, &args, /* No interruption supported */ NULL, NULL);
|
83
|
+
|
84
|
+
// This weird corner case can happen if rb_thread_call_without_gvl2 returns immediately due to an interrupt
|
85
|
+
// without ever calling call_serialize_without_gvl. In this situation, we don't have anything to clean up, we can
|
86
|
+
// just return.
|
87
|
+
if (!args.serialize_ran) {
|
88
|
+
return rb_ary_new_from_args(2, error_symbol, rb_str_new_cstr("Interrupted before call_serialize_without_gvl ran"));
|
89
|
+
}
|
90
|
+
|
91
|
+
ddprof_ffi_SerializeResult serialized_profile = args.result;
|
92
|
+
|
93
|
+
if (serialized_profile.tag == DDPROF_FFI_SERIALIZE_RESULT_ERR) {
|
94
|
+
VALUE err_details = ruby_string_from_vec_u8(serialized_profile.err);
|
95
|
+
ddprof_ffi_SerializeResult_drop(serialized_profile);
|
96
|
+
return rb_ary_new_from_args(2, error_symbol, err_details);
|
97
|
+
}
|
98
|
+
|
99
|
+
VALUE encoded_pprof = ruby_string_from_vec_u8(serialized_profile.ok.buffer);
|
100
|
+
|
101
|
+
ddprof_ffi_Timespec ddprof_start = serialized_profile.ok.start;
|
102
|
+
ddprof_ffi_Timespec ddprof_finish = serialized_profile.ok.end;
|
103
|
+
|
104
|
+
// Clean up libddprof object to avoid leaking in case ruby_time_from raises an exception
|
105
|
+
ddprof_ffi_SerializeResult_drop(serialized_profile);
|
106
|
+
|
107
|
+
VALUE start = ruby_time_from(ddprof_start);
|
108
|
+
VALUE finish = ruby_time_from(ddprof_finish);
|
109
|
+
|
110
|
+
if (!ddprof_ffi_Profile_reset(profile)) return rb_ary_new_from_args(2, error_symbol, rb_str_new_cstr("Failed to reset profile"));
|
111
|
+
|
112
|
+
return rb_ary_new_from_args(2, ok_symbol, rb_ary_new_from_args(3, start, finish, encoded_pprof));
|
113
|
+
}
|
114
|
+
|
115
|
+
static VALUE ruby_time_from(ddprof_ffi_Timespec ddprof_time) {
|
116
|
+
#ifndef NO_RB_TIME_TIMESPEC_NEW // Modern Rubies
|
117
|
+
const int utc = INT_MAX - 1; // From Ruby sources
|
118
|
+
struct timespec time = {.tv_sec = ddprof_time.seconds, .tv_nsec = ddprof_time.nanoseconds};
|
119
|
+
return rb_time_timespec_new(&time, utc);
|
120
|
+
#else // Ruby < 2.3
|
121
|
+
return rb_funcall(stack_recorder_class, ruby_time_from_id, 2, LONG2NUM(ddprof_time.seconds), UINT2NUM(ddprof_time.nanoseconds));
|
122
|
+
#endif
|
123
|
+
}
|
124
|
+
|
125
|
+
void record_sample(VALUE recorder_instance, ddprof_ffi_Sample sample) {
|
126
|
+
ddprof_ffi_Profile *profile;
|
127
|
+
TypedData_Get_Struct(recorder_instance, ddprof_ffi_Profile, &stack_recorder_typed_data, profile);
|
128
|
+
|
129
|
+
ddprof_ffi_Profile_add(profile, sample);
|
130
|
+
}
|
131
|
+
|
132
|
+
static void *call_serialize_without_gvl(void *call_args) {
|
133
|
+
struct call_serialize_without_gvl_arguments *args = (struct call_serialize_without_gvl_arguments *) call_args;
|
134
|
+
|
135
|
+
args->result = ddprof_ffi_Profile_serialize(args->profile);
|
136
|
+
args->serialize_ran = true;
|
137
|
+
|
138
|
+
return NULL; // Unused
|
139
|
+
}
|
@@ -0,0 +1,28 @@
|
|
1
|
+
#pragma once
|
2
|
+
|
3
|
+
#include <ddprof/ffi.h>
|
4
|
+
|
5
|
+
// Note: Please DO NOT use `VALUE_STRING` anywhere else, instead use `DDPROF_FFI_CHARSLICE_C`.
|
6
|
+
// `VALUE_STRING` is only needed because older versions of gcc (4.9.2, used in our Ruby 2.1 and 2.2 CI test images)
|
7
|
+
// tripped when compiling `enabled_value_types` using `-std=gnu99` due to the extra cast that is included in
|
8
|
+
// `DDPROF_FFI_CHARSLICE_C` with the following error:
|
9
|
+
//
|
10
|
+
// ```
|
11
|
+
// compiling ../../../../ext/ddtrace_profiling_native_extension/stack_recorder.c
|
12
|
+
// ../../../../ext/ddtrace_profiling_native_extension/stack_recorder.c:23:1: error: initializer element is not constant
|
13
|
+
// static const ddprof_ffi_ValueType enabled_value_types[] = {CPU_TIME_VALUE, CPU_SAMPLES_VALUE, WALL_TIME_VALUE};
|
14
|
+
// ^
|
15
|
+
// ```
|
16
|
+
#define VALUE_STRING(string) {.ptr = "" string, .len = sizeof(string) - 1}
|
17
|
+
|
18
|
+
#define CPU_TIME_VALUE {.type_ = VALUE_STRING("cpu-time"), .unit = VALUE_STRING("nanoseconds")}
|
19
|
+
#define CPU_SAMPLES_VALUE {.type_ = VALUE_STRING("cpu-samples"), .unit = VALUE_STRING("count")}
|
20
|
+
#define WALL_TIME_VALUE {.type_ = VALUE_STRING("wall-time"), .unit = VALUE_STRING("nanoseconds")}
|
21
|
+
#define ALLOC_SAMPLES_VALUE {.type_ = VALUE_STRING("alloc-samples"), .unit = VALUE_STRING("count")}
|
22
|
+
#define ALLOC_SPACE_VALUE {.type_ = VALUE_STRING("alloc-space"), .unit = VALUE_STRING("bytes")}
|
23
|
+
#define HEAP_SPACE_VALUE {.type_ = VALUE_STRING("heap-space"), .unit = VALUE_STRING("bytes")}
|
24
|
+
|
25
|
+
static const ddprof_ffi_ValueType enabled_value_types[] = {CPU_TIME_VALUE, CPU_SAMPLES_VALUE, WALL_TIME_VALUE};
|
26
|
+
#define ENABLED_VALUE_TYPES_COUNT (sizeof(enabled_value_types) / sizeof(ddprof_ffi_ValueType))
|
27
|
+
|
28
|
+
void record_sample(VALUE recorder_instance, ddprof_ffi_Sample sample);
|
@@ -4,13 +4,13 @@ if %w[1 true].include?((ENV['DD_APPSEC_ENABLED'] || '').downcase)
|
|
4
4
|
begin
|
5
5
|
require 'datadog/appsec'
|
6
6
|
rescue StandardError => e
|
7
|
-
puts "AppSec failed to load. No security check will be performed. error: #{e.message}"
|
7
|
+
puts "AppSec failed to load. No security check will be performed. error: #{e.class.name} #{e.message}"
|
8
8
|
end
|
9
9
|
|
10
10
|
begin
|
11
11
|
require 'datadog/appsec/contrib/auto_instrument'
|
12
12
|
Datadog::AppSec::Contrib::AutoInstrument.patch_all
|
13
13
|
rescue StandardError => e
|
14
|
-
puts "AppSec failed to instrument. No security check will be performed. error: #{e.message}"
|
14
|
+
puts "AppSec failed to instrument. No security check will be performed. error: #{e.class.name} #{e.message}"
|
15
15
|
end
|
16
16
|
end
|
@@ -5,6 +5,7 @@ module Datadog
|
|
5
5
|
module Configuration
|
6
6
|
# Configuration settings, acting as an integration registry
|
7
7
|
# TODO: as with Configuration, this is a trivial implementation
|
8
|
+
# rubocop:disable Metrics/ClassLength
|
8
9
|
class Settings
|
9
10
|
class << self
|
10
11
|
def boolean
|
@@ -84,12 +85,19 @@ module Datadog
|
|
84
85
|
# rubocop:enable Metrics/MethodLength
|
85
86
|
end
|
86
87
|
|
88
|
+
# rubocop:disable Layout/LineLength
|
89
|
+
DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'.freeze
|
90
|
+
DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'.freeze
|
91
|
+
# rubocop:enable Layout/LineLength
|
92
|
+
|
87
93
|
DEFAULTS = {
|
88
94
|
enabled: false,
|
89
95
|
ruleset: :recommended,
|
90
96
|
waf_timeout: 5_000, # us
|
91
97
|
waf_debug: false,
|
92
98
|
trace_rate_limit: 100, # traces/s
|
99
|
+
obfuscator_key_regex: DEFAULT_OBFUSCATOR_KEY_REGEX,
|
100
|
+
obfuscator_value_regex: DEFAULT_OBFUSCATOR_VALUE_REGEX,
|
93
101
|
}.freeze
|
94
102
|
|
95
103
|
ENVS = {
|
@@ -98,6 +106,8 @@ module Datadog
|
|
98
106
|
'DD_APPSEC_WAF_TIMEOUT' => [:waf_timeout, Settings.duration(:us)],
|
99
107
|
'DD_APPSEC_WAF_DEBUG' => [:waf_debug, Settings.boolean],
|
100
108
|
'DD_APPSEC_TRACE_RATE_LIMIT' => [:trace_rate_limit, Settings.integer],
|
109
|
+
'DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP' => [:obfuscator_key_regex, Settings.string],
|
110
|
+
'DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP' => [:obfuscator_value_regex, Settings.string],
|
101
111
|
}.freeze
|
102
112
|
|
103
113
|
Integration = Struct.new(:integration, :options)
|
@@ -131,6 +141,14 @@ module Datadog
|
|
131
141
|
@options[:trace_rate_limit]
|
132
142
|
end
|
133
143
|
|
144
|
+
def obfuscator_key_regex
|
145
|
+
@options[:obfuscator_key_regex]
|
146
|
+
end
|
147
|
+
|
148
|
+
def obfuscator_value_regex
|
149
|
+
@options[:obfuscator_value_regex]
|
150
|
+
end
|
151
|
+
|
134
152
|
def [](integration_name)
|
135
153
|
integration = Datadog::AppSec::Contrib::Integration.registry[integration_name]
|
136
154
|
|
@@ -170,6 +188,7 @@ module Datadog
|
|
170
188
|
initialize
|
171
189
|
end
|
172
190
|
end
|
191
|
+
# rubocop:enable Metrics/ClassLength
|
173
192
|
end
|
174
193
|
end
|
175
194
|
end
|
@@ -47,6 +47,14 @@ module Datadog
|
|
47
47
|
options[:trace_rate_limit] = value
|
48
48
|
end
|
49
49
|
|
50
|
+
def obfuscator_key_regex=(value)
|
51
|
+
options[:obfuscator_key_regex] = value
|
52
|
+
end
|
53
|
+
|
54
|
+
def obfuscator_value_regex=(value)
|
55
|
+
options[:obfuscator_value_regex] = value
|
56
|
+
end
|
57
|
+
|
50
58
|
def [](key)
|
51
59
|
found = @instruments.find { |e| e.name == key }
|
52
60
|
|
@@ -3,6 +3,7 @@
|
|
3
3
|
require 'datadog/appsec/instrumentation/gateway'
|
4
4
|
require 'datadog/appsec/reactive/operation'
|
5
5
|
require 'datadog/appsec/contrib/rack/reactive/request'
|
6
|
+
require 'datadog/appsec/contrib/rack/reactive/request_body'
|
6
7
|
require 'datadog/appsec/contrib/rack/reactive/response'
|
7
8
|
require 'datadog/appsec/event'
|
8
9
|
|
@@ -12,11 +13,10 @@ module Datadog
|
|
12
13
|
module Rack
|
13
14
|
module Gateway
|
14
15
|
# Watcher for Rack gateway events
|
16
|
+
# rubocop:disable Metrics/ModuleLength
|
15
17
|
module Watcher
|
16
18
|
# rubocop:disable Metrics/AbcSize
|
17
|
-
# rubocop:disable Metrics/CyclomaticComplexity
|
18
19
|
# rubocop:disable Metrics/MethodLength
|
19
|
-
# rubocop:disable Metrics/PerceivedComplexity
|
20
20
|
def self.watch
|
21
21
|
Instrumentation.gateway.watch('rack.request') do |stack, request|
|
22
22
|
block = false
|
@@ -24,18 +24,8 @@ module Datadog
|
|
24
24
|
waf_context = request.env['datadog.waf.context']
|
25
25
|
|
26
26
|
AppSec::Reactive::Operation.new('rack.request') do |op|
|
27
|
-
|
28
|
-
|
29
|
-
active_trace = Datadog::Tracing.active_trace
|
30
|
-
active_span = Datadog::Tracing.active_span
|
31
|
-
|
32
|
-
Datadog.logger.debug { "active span: #{active_span.span_id}" } if active_span
|
33
|
-
|
34
|
-
if active_span
|
35
|
-
active_span.set_tag('_dd.appsec.enabled', 1)
|
36
|
-
active_span.set_tag('_dd.runtime_family', 'ruby')
|
37
|
-
end
|
38
|
-
end
|
27
|
+
trace = active_trace
|
28
|
+
span = active_span
|
39
29
|
|
40
30
|
Rack::Reactive::Request.subscribe(op, waf_context) do |action, result, _block|
|
41
31
|
record = [:block, :monitor].include?(action)
|
@@ -43,11 +33,13 @@ module Datadog
|
|
43
33
|
# TODO: should this hash be an Event instance instead?
|
44
34
|
event = {
|
45
35
|
waf_result: result,
|
46
|
-
trace:
|
47
|
-
span:
|
36
|
+
trace: trace,
|
37
|
+
span: span,
|
48
38
|
request: request,
|
49
39
|
action: action
|
50
40
|
}
|
41
|
+
|
42
|
+
waf_context.events << event
|
51
43
|
end
|
52
44
|
end
|
53
45
|
|
@@ -72,18 +64,8 @@ module Datadog
|
|
72
64
|
waf_context = response.instance_eval { @waf_context }
|
73
65
|
|
74
66
|
AppSec::Reactive::Operation.new('rack.response') do |op|
|
75
|
-
|
76
|
-
|
77
|
-
active_trace = Datadog::Tracing.active_trace
|
78
|
-
active_span = Datadog::Tracing.active_span
|
79
|
-
|
80
|
-
Datadog.logger.debug { "active span: #{active_span.span_id}" } if active_span
|
81
|
-
|
82
|
-
if active_span
|
83
|
-
active_span.set_tag('_dd.appsec.enabled', 1)
|
84
|
-
active_span.set_tag('_dd.runtime_family', 'ruby')
|
85
|
-
end
|
86
|
-
end
|
67
|
+
trace = active_trace
|
68
|
+
span = active_span
|
87
69
|
|
88
70
|
Rack::Reactive::Response.subscribe(op, waf_context) do |action, result, _block|
|
89
71
|
record = [:block, :monitor].include?(action)
|
@@ -91,11 +73,13 @@ module Datadog
|
|
91
73
|
# TODO: should this hash be an Event instance instead?
|
92
74
|
event = {
|
93
75
|
waf_result: result,
|
94
|
-
trace:
|
95
|
-
span:
|
76
|
+
trace: trace,
|
77
|
+
span: span,
|
96
78
|
response: response,
|
97
79
|
action: action
|
98
80
|
}
|
81
|
+
|
82
|
+
waf_context.events << event
|
99
83
|
end
|
100
84
|
end
|
101
85
|
|
@@ -113,12 +97,71 @@ module Datadog
|
|
113
97
|
|
114
98
|
[ret, res]
|
115
99
|
end
|
100
|
+
|
101
|
+
Instrumentation.gateway.watch('rack.request.body') do |stack, request|
|
102
|
+
block = false
|
103
|
+
event = nil
|
104
|
+
waf_context = request.env['datadog.waf.context']
|
105
|
+
|
106
|
+
AppSec::Reactive::Operation.new('rack.request.body') do |op|
|
107
|
+
trace = active_trace
|
108
|
+
span = active_span
|
109
|
+
|
110
|
+
Rack::Reactive::RequestBody.subscribe(op, waf_context) do |action, result, _block|
|
111
|
+
record = [:block, :monitor].include?(action)
|
112
|
+
if record
|
113
|
+
# TODO: should this hash be an Event instance instead?
|
114
|
+
event = {
|
115
|
+
waf_result: result,
|
116
|
+
trace: trace,
|
117
|
+
span: span,
|
118
|
+
request: request,
|
119
|
+
action: action
|
120
|
+
}
|
121
|
+
|
122
|
+
waf_context.events << event
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
_action, _result, block = Rack::Reactive::RequestBody.publish(op, request)
|
127
|
+
end
|
128
|
+
|
129
|
+
next [nil, [[:block, event]]] if block
|
130
|
+
|
131
|
+
ret, res = stack.call(request)
|
132
|
+
|
133
|
+
if event
|
134
|
+
res ||= []
|
135
|
+
res << [:monitor, event]
|
136
|
+
end
|
137
|
+
|
138
|
+
[ret, res]
|
139
|
+
end
|
116
140
|
end
|
117
|
-
# rubocop:enable Metrics/AbcSize
|
118
|
-
# rubocop:enable Metrics/CyclomaticComplexity
|
119
141
|
# rubocop:enable Metrics/MethodLength
|
120
|
-
# rubocop:enable Metrics/
|
142
|
+
# rubocop:enable Metrics/AbcSize
|
143
|
+
|
144
|
+
class << self
|
145
|
+
private
|
146
|
+
|
147
|
+
def active_trace
|
148
|
+
# TODO: factor out tracing availability detection
|
149
|
+
|
150
|
+
return unless defined?(Datadog::Tracing)
|
151
|
+
|
152
|
+
Datadog::Tracing.active_trace
|
153
|
+
end
|
154
|
+
|
155
|
+
def active_span
|
156
|
+
# TODO: factor out tracing availability detection
|
157
|
+
|
158
|
+
return unless defined?(Datadog::Tracing)
|
159
|
+
|
160
|
+
Datadog::Tracing.active_span
|
161
|
+
end
|
162
|
+
end
|
121
163
|
end
|
164
|
+
# rubocop:enable Metrics/ModuleLength
|
122
165
|
end
|
123
166
|
end
|
124
167
|
end
|
@@ -5,6 +5,7 @@ require 'datadog/appsec/contrib/integration'
|
|
5
5
|
require 'datadog/appsec/contrib/rack/configuration/settings'
|
6
6
|
require 'datadog/appsec/contrib/rack/patcher'
|
7
7
|
require 'datadog/appsec/contrib/rack/request_middleware'
|
8
|
+
require 'datadog/appsec/contrib/rack/request_body_middleware'
|
8
9
|
|
9
10
|
module Datadog
|
10
11
|
module AppSec
|
@@ -0,0 +1,64 @@
|
|
1
|
+
# typed: true
|
2
|
+
|
3
|
+
require 'datadog/appsec/contrib/rack/request'
|
4
|
+
|
5
|
+
module Datadog
|
6
|
+
module AppSec
|
7
|
+
module Contrib
|
8
|
+
module Rack
|
9
|
+
module Reactive
|
10
|
+
# Dispatch data from a Rack request to the WAF context
|
11
|
+
module RequestBody
|
12
|
+
def self.publish(op, request)
|
13
|
+
catch(:block) do
|
14
|
+
# params have been parsed from the request body
|
15
|
+
op.publish('request.body', Rack::Request.form_hash(request))
|
16
|
+
|
17
|
+
nil
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.subscribe(op, waf_context)
|
22
|
+
addresses = [
|
23
|
+
'request.body',
|
24
|
+
]
|
25
|
+
|
26
|
+
op.subscribe(*addresses) do |*values|
|
27
|
+
Datadog.logger.debug { "reacted to #{addresses.inspect}: #{values.inspect}" }
|
28
|
+
body = values[0]
|
29
|
+
|
30
|
+
waf_args = {
|
31
|
+
'server.request.body' => body,
|
32
|
+
}
|
33
|
+
|
34
|
+
waf_timeout = Datadog::AppSec.settings.waf_timeout
|
35
|
+
action, result = waf_context.run(waf_args, waf_timeout)
|
36
|
+
|
37
|
+
Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
|
38
|
+
|
39
|
+
# TODO: encapsulate return array in a type
|
40
|
+
case action
|
41
|
+
when :monitor
|
42
|
+
Datadog.logger.debug { "WAF: #{result.inspect}" }
|
43
|
+
yield [action, result, false]
|
44
|
+
when :block
|
45
|
+
Datadog.logger.debug { "WAF: #{result.inspect}" }
|
46
|
+
yield [action, result, true]
|
47
|
+
throw(:block, [action, result, true])
|
48
|
+
when :good
|
49
|
+
Datadog.logger.debug { "WAF OK: #{result.inspect}" }
|
50
|
+
when :invalid_call
|
51
|
+
Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
|
52
|
+
when :invalid_rule, :invalid_flow, :no_rule
|
53
|
+
Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
|
54
|
+
else
|
55
|
+
Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -45,6 +45,12 @@ module Datadog
|
|
45
45
|
def self.cookies(request)
|
46
46
|
request.cookies
|
47
47
|
end
|
48
|
+
|
49
|
+
def self.form_hash(request)
|
50
|
+
# usually Hash<String,String> but can be a more complex
|
51
|
+
# Hash<String,String||Array||Hash> when e.g coming from JSON
|
52
|
+
request.env['rack.request.form_hash']
|
53
|
+
end
|
48
54
|
end
|
49
55
|
end
|
50
56
|
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# typed: ignore
|
2
|
+
|
3
|
+
require 'datadog/appsec/instrumentation/gateway'
|
4
|
+
require 'datadog/appsec/assets'
|
5
|
+
|
6
|
+
module Datadog
|
7
|
+
module AppSec
|
8
|
+
module Contrib
|
9
|
+
module Rack
|
10
|
+
# Rack request body middleware for AppSec
|
11
|
+
# This should be inserted just below Rack::JSONBodyParser or
|
12
|
+
# legacy Rack::PostBodyContentTypeParser from rack-contrib
|
13
|
+
class RequestBodyMiddleware
|
14
|
+
def initialize(app, opt = {})
|
15
|
+
@app = app
|
16
|
+
end
|
17
|
+
|
18
|
+
def call(env)
|
19
|
+
context = env['datadog.waf.context']
|
20
|
+
|
21
|
+
return @app.call(env) unless context
|
22
|
+
|
23
|
+
# TODO: handle exceptions, except for @app.call
|
24
|
+
|
25
|
+
request = ::Rack::Request.new(env)
|
26
|
+
|
27
|
+
request_return, request_response = Instrumentation.gateway.push('rack.request.body', request) do
|
28
|
+
@app.call(env)
|
29
|
+
end
|
30
|
+
|
31
|
+
if request_response && request_response.any? { |action, _event| action == :block }
|
32
|
+
request_return = [403, { 'Content-Type' => 'text/html' }, [Datadog::AppSec::Assets.blocked]]
|
33
|
+
end
|
34
|
+
|
35
|
+
request_return
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# typed: ignore
|
2
2
|
|
3
|
+
require 'json'
|
4
|
+
|
3
5
|
require 'datadog/appsec/instrumentation/gateway'
|
4
6
|
require 'datadog/appsec/processor'
|
5
7
|
require 'datadog/appsec/assets'
|
@@ -14,6 +16,7 @@ module Datadog
|
|
14
16
|
def initialize(app, opt = {})
|
15
17
|
@app = app
|
16
18
|
|
19
|
+
@oneshot_tags_sent = false
|
17
20
|
@processor = Datadog::AppSec::Processor.new
|
18
21
|
end
|
19
22
|
|
@@ -27,6 +30,8 @@ module Datadog
|
|
27
30
|
env['datadog.waf.context'] = context
|
28
31
|
request = ::Rack::Request.new(env)
|
29
32
|
|
33
|
+
add_appsec_tags
|
34
|
+
|
30
35
|
request_return, request_response = Instrumentation.gateway.push('rack.request', request) do
|
31
36
|
@app.call(env)
|
32
37
|
end
|
@@ -40,15 +45,65 @@ module Datadog
|
|
40
45
|
@waf_context = context
|
41
46
|
end
|
42
47
|
|
43
|
-
|
48
|
+
_response_return, _response_response = Instrumentation.gateway.push('rack.response', response)
|
44
49
|
|
45
|
-
|
46
|
-
|
47
|
-
|
50
|
+
context.events.each do |e|
|
51
|
+
e[:response] ||= response
|
52
|
+
e[:request] ||= request
|
53
|
+
end
|
48
54
|
|
49
|
-
AppSec::Event.record(*
|
55
|
+
AppSec::Event.record(*context.events)
|
50
56
|
|
51
57
|
request_return
|
58
|
+
ensure
|
59
|
+
add_waf_runtime_tags(context) if context
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
def active_trace
|
65
|
+
# TODO: factor out tracing availability detection
|
66
|
+
|
67
|
+
return unless defined?(Datadog::Tracing)
|
68
|
+
|
69
|
+
Datadog::Tracing.active_trace
|
70
|
+
end
|
71
|
+
|
72
|
+
def add_appsec_tags
|
73
|
+
return unless active_trace
|
74
|
+
|
75
|
+
active_trace.set_tag('_dd.appsec.enabled', 1)
|
76
|
+
active_trace.set_tag('_dd.runtime_family', 'ruby')
|
77
|
+
active_trace.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
|
78
|
+
|
79
|
+
if @processor.ruleset_info
|
80
|
+
active_trace.set_tag('_dd.appsec.event_rules.version', @processor.ruleset_info[:version])
|
81
|
+
|
82
|
+
unless @oneshot_tags_sent
|
83
|
+
# Small race condition, but it's inoccuous: worst case the tags
|
84
|
+
# are sent a couple of times more than expected
|
85
|
+
@oneshot_tags_sent = true
|
86
|
+
|
87
|
+
active_trace.set_tag('_dd.appsec.event_rules.loaded', @processor.ruleset_info[:loaded].to_f)
|
88
|
+
active_trace.set_tag('_dd.appsec.event_rules.error_count', @processor.ruleset_info[:failed].to_f)
|
89
|
+
active_trace.set_tag('_dd.appsec.event_rules.errors', JSON.dump(@processor.ruleset_info[:errors]))
|
90
|
+
active_trace.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(@processor.addresses))
|
91
|
+
|
92
|
+
# Ensure these tags reach the backend
|
93
|
+
active_trace.keep!
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def add_waf_runtime_tags(context)
|
99
|
+
return unless active_trace
|
100
|
+
return unless context
|
101
|
+
|
102
|
+
active_trace.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
|
103
|
+
|
104
|
+
# these tags expect time in us
|
105
|
+
active_trace.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
|
106
|
+
active_trace.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
|
52
107
|
end
|
53
108
|
end
|
54
109
|
end
|