ddtrace 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +4 -16
- data/CHANGELOG.md +31 -2
- data/LICENSE-3rdparty.csv +3 -2
- data/README.md +2 -2
- data/ddtrace.gemspec +12 -3
- data/docs/GettingStarted.md +19 -2
- data/docs/ProfilingDevelopment.md +8 -8
- data/docs/UpgradeGuide.md +3 -3
- data/ext/ddtrace_profiling_loader/ddtrace_profiling_loader.c +118 -0
- data/ext/ddtrace_profiling_loader/extconf.rb +53 -0
- data/ext/ddtrace_profiling_native_extension/NativeExtensionDesign.md +31 -5
- data/ext/ddtrace_profiling_native_extension/clock_id_from_pthread.c +0 -8
- data/ext/ddtrace_profiling_native_extension/collectors_stack.c +278 -0
- data/ext/ddtrace_profiling_native_extension/extconf.rb +70 -100
- data/ext/ddtrace_profiling_native_extension/libddprof_helpers.h +13 -0
- data/ext/ddtrace_profiling_native_extension/native_extension_helpers.rb +186 -0
- data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c +579 -7
- data/ext/ddtrace_profiling_native_extension/private_vm_api_access.h +30 -0
- data/ext/ddtrace_profiling_native_extension/profiling.c +7 -0
- data/ext/ddtrace_profiling_native_extension/stack_recorder.c +139 -0
- data/ext/ddtrace_profiling_native_extension/stack_recorder.h +28 -0
- data/lib/datadog/appsec/autoload.rb +2 -2
- data/lib/datadog/appsec/configuration/settings.rb +19 -0
- data/lib/datadog/appsec/configuration.rb +8 -0
- data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb +76 -33
- data/lib/datadog/appsec/contrib/rack/integration.rb +1 -0
- data/lib/datadog/appsec/contrib/rack/patcher.rb +0 -1
- data/lib/datadog/appsec/contrib/rack/reactive/request_body.rb +64 -0
- data/lib/datadog/appsec/contrib/rack/request.rb +6 -0
- data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb +41 -0
- data/lib/datadog/appsec/contrib/rack/request_middleware.rb +60 -5
- data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb +81 -0
- data/lib/datadog/appsec/contrib/rails/patcher.rb +34 -1
- data/lib/datadog/appsec/contrib/rails/reactive/action.rb +68 -0
- data/lib/datadog/appsec/contrib/rails/request.rb +33 -0
- data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +124 -0
- data/lib/datadog/appsec/contrib/sinatra/patcher.rb +69 -2
- data/lib/datadog/appsec/contrib/sinatra/reactive/routed.rb +63 -0
- data/lib/datadog/appsec/event.rb +33 -18
- data/lib/datadog/appsec/extensions.rb +0 -3
- data/lib/datadog/appsec/processor.rb +45 -2
- data/lib/datadog/appsec/rate_limiter.rb +5 -0
- data/lib/datadog/appsec/reactive/operation.rb +0 -1
- data/lib/datadog/ci/ext/environment.rb +21 -7
- data/lib/datadog/core/configuration/agent_settings_resolver.rb +1 -1
- data/lib/datadog/core/configuration/components.rb +22 -4
- data/lib/datadog/core/configuration/settings.rb +3 -3
- data/lib/datadog/core/configuration.rb +7 -5
- data/lib/datadog/core/environment/cgroup.rb +3 -1
- data/lib/datadog/core/environment/container.rb +2 -1
- data/lib/datadog/core/environment/variable_helpers.rb +26 -2
- data/lib/datadog/core/logging/ext.rb +11 -0
- data/lib/datadog/core/metrics/client.rb +15 -5
- data/lib/datadog/core/runtime/metrics.rb +1 -1
- data/lib/datadog/core/workers/async.rb +3 -1
- data/lib/datadog/core/workers/runtime_metrics.rb +0 -3
- data/lib/datadog/core.rb +6 -0
- data/lib/datadog/kit/enable_core_dumps.rb +50 -0
- data/lib/datadog/kit/identity.rb +63 -0
- data/lib/datadog/kit.rb +11 -0
- data/lib/datadog/opentracer/tracer.rb +0 -2
- data/lib/datadog/profiling/collectors/old_stack.rb +298 -0
- data/lib/datadog/profiling/collectors/stack.rb +6 -287
- data/lib/datadog/profiling/encoding/profile.rb +0 -1
- data/lib/datadog/profiling/ext.rb +1 -1
- data/lib/datadog/profiling/flush.rb +1 -1
- data/lib/datadog/profiling/load_native_extension.rb +22 -0
- data/lib/datadog/profiling/recorder.rb +1 -1
- data/lib/datadog/profiling/scheduler.rb +1 -1
- data/lib/datadog/profiling/stack_recorder.rb +33 -0
- data/lib/datadog/profiling/tag_builder.rb +48 -0
- data/lib/datadog/profiling/tasks/exec.rb +2 -2
- data/lib/datadog/profiling/tasks/setup.rb +6 -4
- data/lib/datadog/profiling.rb +29 -27
- data/lib/datadog/tracing/buffer.rb +9 -3
- data/lib/datadog/tracing/contrib/action_view/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/active_record/configuration/resolver.rb +2 -2
- data/lib/datadog/tracing/contrib/active_record/utils.rb +1 -1
- data/lib/datadog/tracing/contrib/active_record/vendor/connection_specification.rb +1 -1
- data/lib/datadog/tracing/contrib/active_support/notifications/subscription.rb +4 -2
- data/lib/datadog/tracing/contrib/concurrent_ruby/context_composite_executor_service.rb +10 -3
- data/lib/datadog/tracing/contrib/dalli/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/delayed_job/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/elasticsearch/integration.rb +9 -3
- data/lib/datadog/tracing/contrib/elasticsearch/patcher.rb +38 -2
- data/lib/datadog/tracing/contrib/ethon/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/extensions.rb +0 -2
- data/lib/datadog/tracing/contrib/faraday/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/grape/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/graphql/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/grpc/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/kafka/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/lograge/instrumentation.rb +2 -1
- data/lib/datadog/tracing/contrib/qless/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/que/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/racecar/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/rails/log_injection.rb +3 -16
- data/lib/datadog/tracing/contrib/rake/instrumentation.rb +2 -2
- data/lib/datadog/tracing/contrib/rake/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/redis/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/resque/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/rest_client/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/semantic_logger/instrumentation.rb +2 -1
- data/lib/datadog/tracing/contrib/sidekiq/configuration/settings.rb +1 -0
- data/lib/datadog/tracing/contrib/sidekiq/server_tracer.rb +20 -1
- data/lib/datadog/tracing/contrib/sinatra/framework.rb +11 -0
- data/lib/datadog/tracing/contrib/sinatra/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/sneakers/patcher.rb +0 -1
- data/lib/datadog/tracing/contrib/sucker_punch/patcher.rb +0 -1
- data/lib/datadog/tracing/event.rb +2 -1
- data/lib/datadog/tracing/sampling/priority_sampler.rb +4 -5
- data/lib/datadog/tracing/sampling/rule.rb +12 -6
- data/lib/datadog/tracing/sampling/rule_sampler.rb +3 -5
- data/lib/datadog/tracing/span_operation.rb +2 -3
- data/lib/datadog/tracing/trace_operation.rb +0 -1
- data/lib/ddtrace/transport/http/client.rb +2 -1
- data/lib/ddtrace/transport/http/response.rb +34 -4
- data/lib/ddtrace/transport/io/client.rb +3 -1
- data/lib/ddtrace/version.rb +1 -1
- data/lib/ddtrace.rb +1 -0
- metadata +43 -6
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
#include <ruby.h>
|
|
2
|
+
#include <ruby/thread.h>
|
|
3
|
+
#include "stack_recorder.h"
|
|
4
|
+
#include "libddprof_helpers.h"
|
|
5
|
+
|
|
6
|
+
// Used to wrap a ddprof_ffi_Profile in a Ruby object and expose Ruby-level serialization APIs
|
|
7
|
+
// This file implements the native bits of the Datadog::Profiling::StackRecorder class
|
|
8
|
+
|
|
9
|
+
static VALUE ok_symbol = Qnil; // :ok in Ruby
|
|
10
|
+
static VALUE error_symbol = Qnil; // :error in Ruby
|
|
11
|
+
|
|
12
|
+
static ID ruby_time_from_id; // id of :ruby_time_from in Ruby
|
|
13
|
+
|
|
14
|
+
static VALUE stack_recorder_class = Qnil;
|
|
15
|
+
|
|
16
|
+
struct call_serialize_without_gvl_arguments {
|
|
17
|
+
ddprof_ffi_Profile *profile;
|
|
18
|
+
ddprof_ffi_SerializeResult result;
|
|
19
|
+
bool serialize_ran;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
static VALUE _native_new(VALUE klass);
|
|
23
|
+
static void stack_recorder_typed_data_free(void *data);
|
|
24
|
+
static VALUE _native_serialize(VALUE self, VALUE recorder_instance);
|
|
25
|
+
static VALUE ruby_time_from(ddprof_ffi_Timespec ddprof_time);
|
|
26
|
+
static void *call_serialize_without_gvl(void *call_args);
|
|
27
|
+
|
|
28
|
+
void stack_recorder_init(VALUE profiling_module) {
|
|
29
|
+
stack_recorder_class = rb_define_class_under(profiling_module, "StackRecorder", rb_cObject);
|
|
30
|
+
|
|
31
|
+
// Instances of the StackRecorder class are going to be "TypedData" objects.
|
|
32
|
+
// "TypedData" objects are special objects in the Ruby VM that can wrap C structs.
|
|
33
|
+
// In our case, we're going to keep a libddprof profile reference inside our object.
|
|
34
|
+
//
|
|
35
|
+
// Because Ruby doesn't know how to initialize libddprof profiles, we MUST override the allocation function for objects
|
|
36
|
+
// of this class so that we can manage this part. Not overriding or disabling the allocation function is a common
|
|
37
|
+
// gotcha for "TypedData" objects that can very easily lead to VM crashes, see for instance
|
|
38
|
+
// https://bugs.ruby-lang.org/issues/18007 for a discussion around this.
|
|
39
|
+
rb_define_alloc_func(stack_recorder_class, _native_new);
|
|
40
|
+
|
|
41
|
+
rb_define_singleton_method(stack_recorder_class, "_native_serialize", _native_serialize, 1);
|
|
42
|
+
|
|
43
|
+
ok_symbol = ID2SYM(rb_intern_const("ok"));
|
|
44
|
+
error_symbol = ID2SYM(rb_intern_const("error"));
|
|
45
|
+
ruby_time_from_id = rb_intern_const("ruby_time_from");
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// This structure is used to define a Ruby object that stores a pointer to a ddprof_ffi_Profile instance
|
|
49
|
+
// See also https://github.com/ruby/ruby/blob/master/doc/extension.rdoc for how this works
|
|
50
|
+
static const rb_data_type_t stack_recorder_typed_data = {
|
|
51
|
+
.wrap_struct_name = "Datadog::Profiling::StackRecorder",
|
|
52
|
+
.function = {
|
|
53
|
+
.dfree = stack_recorder_typed_data_free,
|
|
54
|
+
.dsize = NULL, // We don't track profile memory usage (although it'd be cool if we did!)
|
|
55
|
+
// No need to provide dmark nor dcompact because we don't directly reference Ruby VALUEs from inside this object
|
|
56
|
+
},
|
|
57
|
+
.flags = RUBY_TYPED_FREE_IMMEDIATELY
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
static VALUE _native_new(VALUE klass) {
|
|
61
|
+
ddprof_ffi_Slice_value_type sample_types = {.ptr = enabled_value_types, .len = ENABLED_VALUE_TYPES_COUNT};
|
|
62
|
+
|
|
63
|
+
ddprof_ffi_Profile *profile = ddprof_ffi_Profile_new(sample_types, NULL /* Period is optional */);
|
|
64
|
+
|
|
65
|
+
return TypedData_Wrap_Struct(klass, &stack_recorder_typed_data, profile);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
static void stack_recorder_typed_data_free(void *data) {
|
|
69
|
+
ddprof_ffi_Profile_free((ddprof_ffi_Profile *) data);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
static VALUE _native_serialize(VALUE self, VALUE recorder_instance) {
|
|
73
|
+
ddprof_ffi_Profile *profile;
|
|
74
|
+
TypedData_Get_Struct(recorder_instance, ddprof_ffi_Profile, &stack_recorder_typed_data, profile);
|
|
75
|
+
|
|
76
|
+
// We'll release the Global VM Lock while we're calling serialize, so that the Ruby VM can continue to work while this
|
|
77
|
+
// is pending
|
|
78
|
+
struct call_serialize_without_gvl_arguments args = {.profile = profile, .serialize_ran = false};
|
|
79
|
+
|
|
80
|
+
// We use rb_thread_call_without_gvl2 for similar reasons as in http_transport.c: we don't want pending interrupts
|
|
81
|
+
// that cause exceptions to be raised to be processed as otherwise we can leak the serialized profile.
|
|
82
|
+
rb_thread_call_without_gvl2(call_serialize_without_gvl, &args, /* No interruption supported */ NULL, NULL);
|
|
83
|
+
|
|
84
|
+
// This weird corner case can happen if rb_thread_call_without_gvl2 returns immediately due to an interrupt
|
|
85
|
+
// without ever calling call_serialize_without_gvl. In this situation, we don't have anything to clean up, we can
|
|
86
|
+
// just return.
|
|
87
|
+
if (!args.serialize_ran) {
|
|
88
|
+
return rb_ary_new_from_args(2, error_symbol, rb_str_new_cstr("Interrupted before call_serialize_without_gvl ran"));
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
ddprof_ffi_SerializeResult serialized_profile = args.result;
|
|
92
|
+
|
|
93
|
+
if (serialized_profile.tag == DDPROF_FFI_SERIALIZE_RESULT_ERR) {
|
|
94
|
+
VALUE err_details = ruby_string_from_vec_u8(serialized_profile.err);
|
|
95
|
+
ddprof_ffi_SerializeResult_drop(serialized_profile);
|
|
96
|
+
return rb_ary_new_from_args(2, error_symbol, err_details);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
VALUE encoded_pprof = ruby_string_from_vec_u8(serialized_profile.ok.buffer);
|
|
100
|
+
|
|
101
|
+
ddprof_ffi_Timespec ddprof_start = serialized_profile.ok.start;
|
|
102
|
+
ddprof_ffi_Timespec ddprof_finish = serialized_profile.ok.end;
|
|
103
|
+
|
|
104
|
+
// Clean up libddprof object to avoid leaking in case ruby_time_from raises an exception
|
|
105
|
+
ddprof_ffi_SerializeResult_drop(serialized_profile);
|
|
106
|
+
|
|
107
|
+
VALUE start = ruby_time_from(ddprof_start);
|
|
108
|
+
VALUE finish = ruby_time_from(ddprof_finish);
|
|
109
|
+
|
|
110
|
+
if (!ddprof_ffi_Profile_reset(profile)) return rb_ary_new_from_args(2, error_symbol, rb_str_new_cstr("Failed to reset profile"));
|
|
111
|
+
|
|
112
|
+
return rb_ary_new_from_args(2, ok_symbol, rb_ary_new_from_args(3, start, finish, encoded_pprof));
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
static VALUE ruby_time_from(ddprof_ffi_Timespec ddprof_time) {
|
|
116
|
+
#ifndef NO_RB_TIME_TIMESPEC_NEW // Modern Rubies
|
|
117
|
+
const int utc = INT_MAX - 1; // From Ruby sources
|
|
118
|
+
struct timespec time = {.tv_sec = ddprof_time.seconds, .tv_nsec = ddprof_time.nanoseconds};
|
|
119
|
+
return rb_time_timespec_new(&time, utc);
|
|
120
|
+
#else // Ruby < 2.3
|
|
121
|
+
return rb_funcall(stack_recorder_class, ruby_time_from_id, 2, LONG2NUM(ddprof_time.seconds), UINT2NUM(ddprof_time.nanoseconds));
|
|
122
|
+
#endif
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
void record_sample(VALUE recorder_instance, ddprof_ffi_Sample sample) {
|
|
126
|
+
ddprof_ffi_Profile *profile;
|
|
127
|
+
TypedData_Get_Struct(recorder_instance, ddprof_ffi_Profile, &stack_recorder_typed_data, profile);
|
|
128
|
+
|
|
129
|
+
ddprof_ffi_Profile_add(profile, sample);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
static void *call_serialize_without_gvl(void *call_args) {
|
|
133
|
+
struct call_serialize_without_gvl_arguments *args = (struct call_serialize_without_gvl_arguments *) call_args;
|
|
134
|
+
|
|
135
|
+
args->result = ddprof_ffi_Profile_serialize(args->profile);
|
|
136
|
+
args->serialize_ran = true;
|
|
137
|
+
|
|
138
|
+
return NULL; // Unused
|
|
139
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include <ddprof/ffi.h>
|
|
4
|
+
|
|
5
|
+
// Note: Please DO NOT use `VALUE_STRING` anywhere else, instead use `DDPROF_FFI_CHARSLICE_C`.
|
|
6
|
+
// `VALUE_STRING` is only needed because older versions of gcc (4.9.2, used in our Ruby 2.1 and 2.2 CI test images)
|
|
7
|
+
// tripped when compiling `enabled_value_types` using `-std=gnu99` due to the extra cast that is included in
|
|
8
|
+
// `DDPROF_FFI_CHARSLICE_C` with the following error:
|
|
9
|
+
//
|
|
10
|
+
// ```
|
|
11
|
+
// compiling ../../../../ext/ddtrace_profiling_native_extension/stack_recorder.c
|
|
12
|
+
// ../../../../ext/ddtrace_profiling_native_extension/stack_recorder.c:23:1: error: initializer element is not constant
|
|
13
|
+
// static const ddprof_ffi_ValueType enabled_value_types[] = {CPU_TIME_VALUE, CPU_SAMPLES_VALUE, WALL_TIME_VALUE};
|
|
14
|
+
// ^
|
|
15
|
+
// ```
|
|
16
|
+
#define VALUE_STRING(string) {.ptr = "" string, .len = sizeof(string) - 1}
|
|
17
|
+
|
|
18
|
+
#define CPU_TIME_VALUE {.type_ = VALUE_STRING("cpu-time"), .unit = VALUE_STRING("nanoseconds")}
|
|
19
|
+
#define CPU_SAMPLES_VALUE {.type_ = VALUE_STRING("cpu-samples"), .unit = VALUE_STRING("count")}
|
|
20
|
+
#define WALL_TIME_VALUE {.type_ = VALUE_STRING("wall-time"), .unit = VALUE_STRING("nanoseconds")}
|
|
21
|
+
#define ALLOC_SAMPLES_VALUE {.type_ = VALUE_STRING("alloc-samples"), .unit = VALUE_STRING("count")}
|
|
22
|
+
#define ALLOC_SPACE_VALUE {.type_ = VALUE_STRING("alloc-space"), .unit = VALUE_STRING("bytes")}
|
|
23
|
+
#define HEAP_SPACE_VALUE {.type_ = VALUE_STRING("heap-space"), .unit = VALUE_STRING("bytes")}
|
|
24
|
+
|
|
25
|
+
static const ddprof_ffi_ValueType enabled_value_types[] = {CPU_TIME_VALUE, CPU_SAMPLES_VALUE, WALL_TIME_VALUE};
|
|
26
|
+
#define ENABLED_VALUE_TYPES_COUNT (sizeof(enabled_value_types) / sizeof(ddprof_ffi_ValueType))
|
|
27
|
+
|
|
28
|
+
void record_sample(VALUE recorder_instance, ddprof_ffi_Sample sample);
|
|
@@ -4,13 +4,13 @@ if %w[1 true].include?((ENV['DD_APPSEC_ENABLED'] || '').downcase)
|
|
|
4
4
|
begin
|
|
5
5
|
require 'datadog/appsec'
|
|
6
6
|
rescue StandardError => e
|
|
7
|
-
puts "AppSec failed to load. No security check will be performed. error: #{e.message}"
|
|
7
|
+
puts "AppSec failed to load. No security check will be performed. error: #{e.class.name} #{e.message}"
|
|
8
8
|
end
|
|
9
9
|
|
|
10
10
|
begin
|
|
11
11
|
require 'datadog/appsec/contrib/auto_instrument'
|
|
12
12
|
Datadog::AppSec::Contrib::AutoInstrument.patch_all
|
|
13
13
|
rescue StandardError => e
|
|
14
|
-
puts "AppSec failed to instrument. No security check will be performed. error: #{e.message}"
|
|
14
|
+
puts "AppSec failed to instrument. No security check will be performed. error: #{e.class.name} #{e.message}"
|
|
15
15
|
end
|
|
16
16
|
end
|
|
@@ -5,6 +5,7 @@ module Datadog
|
|
|
5
5
|
module Configuration
|
|
6
6
|
# Configuration settings, acting as an integration registry
|
|
7
7
|
# TODO: as with Configuration, this is a trivial implementation
|
|
8
|
+
# rubocop:disable Metrics/ClassLength
|
|
8
9
|
class Settings
|
|
9
10
|
class << self
|
|
10
11
|
def boolean
|
|
@@ -84,12 +85,19 @@ module Datadog
|
|
|
84
85
|
# rubocop:enable Metrics/MethodLength
|
|
85
86
|
end
|
|
86
87
|
|
|
88
|
+
# rubocop:disable Layout/LineLength
|
|
89
|
+
DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'.freeze
|
|
90
|
+
DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'.freeze
|
|
91
|
+
# rubocop:enable Layout/LineLength
|
|
92
|
+
|
|
87
93
|
DEFAULTS = {
|
|
88
94
|
enabled: false,
|
|
89
95
|
ruleset: :recommended,
|
|
90
96
|
waf_timeout: 5_000, # us
|
|
91
97
|
waf_debug: false,
|
|
92
98
|
trace_rate_limit: 100, # traces/s
|
|
99
|
+
obfuscator_key_regex: DEFAULT_OBFUSCATOR_KEY_REGEX,
|
|
100
|
+
obfuscator_value_regex: DEFAULT_OBFUSCATOR_VALUE_REGEX,
|
|
93
101
|
}.freeze
|
|
94
102
|
|
|
95
103
|
ENVS = {
|
|
@@ -98,6 +106,8 @@ module Datadog
|
|
|
98
106
|
'DD_APPSEC_WAF_TIMEOUT' => [:waf_timeout, Settings.duration(:us)],
|
|
99
107
|
'DD_APPSEC_WAF_DEBUG' => [:waf_debug, Settings.boolean],
|
|
100
108
|
'DD_APPSEC_TRACE_RATE_LIMIT' => [:trace_rate_limit, Settings.integer],
|
|
109
|
+
'DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP' => [:obfuscator_key_regex, Settings.string],
|
|
110
|
+
'DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP' => [:obfuscator_value_regex, Settings.string],
|
|
101
111
|
}.freeze
|
|
102
112
|
|
|
103
113
|
Integration = Struct.new(:integration, :options)
|
|
@@ -131,6 +141,14 @@ module Datadog
|
|
|
131
141
|
@options[:trace_rate_limit]
|
|
132
142
|
end
|
|
133
143
|
|
|
144
|
+
def obfuscator_key_regex
|
|
145
|
+
@options[:obfuscator_key_regex]
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def obfuscator_value_regex
|
|
149
|
+
@options[:obfuscator_value_regex]
|
|
150
|
+
end
|
|
151
|
+
|
|
134
152
|
def [](integration_name)
|
|
135
153
|
integration = Datadog::AppSec::Contrib::Integration.registry[integration_name]
|
|
136
154
|
|
|
@@ -170,6 +188,7 @@ module Datadog
|
|
|
170
188
|
initialize
|
|
171
189
|
end
|
|
172
190
|
end
|
|
191
|
+
# rubocop:enable Metrics/ClassLength
|
|
173
192
|
end
|
|
174
193
|
end
|
|
175
194
|
end
|
|
@@ -47,6 +47,14 @@ module Datadog
|
|
|
47
47
|
options[:trace_rate_limit] = value
|
|
48
48
|
end
|
|
49
49
|
|
|
50
|
+
def obfuscator_key_regex=(value)
|
|
51
|
+
options[:obfuscator_key_regex] = value
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def obfuscator_value_regex=(value)
|
|
55
|
+
options[:obfuscator_value_regex] = value
|
|
56
|
+
end
|
|
57
|
+
|
|
50
58
|
def [](key)
|
|
51
59
|
found = @instruments.find { |e| e.name == key }
|
|
52
60
|
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require 'datadog/appsec/instrumentation/gateway'
|
|
4
4
|
require 'datadog/appsec/reactive/operation'
|
|
5
5
|
require 'datadog/appsec/contrib/rack/reactive/request'
|
|
6
|
+
require 'datadog/appsec/contrib/rack/reactive/request_body'
|
|
6
7
|
require 'datadog/appsec/contrib/rack/reactive/response'
|
|
7
8
|
require 'datadog/appsec/event'
|
|
8
9
|
|
|
@@ -12,11 +13,10 @@ module Datadog
|
|
|
12
13
|
module Rack
|
|
13
14
|
module Gateway
|
|
14
15
|
# Watcher for Rack gateway events
|
|
16
|
+
# rubocop:disable Metrics/ModuleLength
|
|
15
17
|
module Watcher
|
|
16
18
|
# rubocop:disable Metrics/AbcSize
|
|
17
|
-
# rubocop:disable Metrics/CyclomaticComplexity
|
|
18
19
|
# rubocop:disable Metrics/MethodLength
|
|
19
|
-
# rubocop:disable Metrics/PerceivedComplexity
|
|
20
20
|
def self.watch
|
|
21
21
|
Instrumentation.gateway.watch('rack.request') do |stack, request|
|
|
22
22
|
block = false
|
|
@@ -24,18 +24,8 @@ module Datadog
|
|
|
24
24
|
waf_context = request.env['datadog.waf.context']
|
|
25
25
|
|
|
26
26
|
AppSec::Reactive::Operation.new('rack.request') do |op|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
active_trace = Datadog::Tracing.active_trace
|
|
30
|
-
active_span = Datadog::Tracing.active_span
|
|
31
|
-
|
|
32
|
-
Datadog.logger.debug { "active span: #{active_span.span_id}" } if active_span
|
|
33
|
-
|
|
34
|
-
if active_span
|
|
35
|
-
active_span.set_tag('_dd.appsec.enabled', 1)
|
|
36
|
-
active_span.set_tag('_dd.runtime_family', 'ruby')
|
|
37
|
-
end
|
|
38
|
-
end
|
|
27
|
+
trace = active_trace
|
|
28
|
+
span = active_span
|
|
39
29
|
|
|
40
30
|
Rack::Reactive::Request.subscribe(op, waf_context) do |action, result, _block|
|
|
41
31
|
record = [:block, :monitor].include?(action)
|
|
@@ -43,11 +33,13 @@ module Datadog
|
|
|
43
33
|
# TODO: should this hash be an Event instance instead?
|
|
44
34
|
event = {
|
|
45
35
|
waf_result: result,
|
|
46
|
-
trace:
|
|
47
|
-
span:
|
|
36
|
+
trace: trace,
|
|
37
|
+
span: span,
|
|
48
38
|
request: request,
|
|
49
39
|
action: action
|
|
50
40
|
}
|
|
41
|
+
|
|
42
|
+
waf_context.events << event
|
|
51
43
|
end
|
|
52
44
|
end
|
|
53
45
|
|
|
@@ -72,18 +64,8 @@ module Datadog
|
|
|
72
64
|
waf_context = response.instance_eval { @waf_context }
|
|
73
65
|
|
|
74
66
|
AppSec::Reactive::Operation.new('rack.response') do |op|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
active_trace = Datadog::Tracing.active_trace
|
|
78
|
-
active_span = Datadog::Tracing.active_span
|
|
79
|
-
|
|
80
|
-
Datadog.logger.debug { "active span: #{active_span.span_id}" } if active_span
|
|
81
|
-
|
|
82
|
-
if active_span
|
|
83
|
-
active_span.set_tag('_dd.appsec.enabled', 1)
|
|
84
|
-
active_span.set_tag('_dd.runtime_family', 'ruby')
|
|
85
|
-
end
|
|
86
|
-
end
|
|
67
|
+
trace = active_trace
|
|
68
|
+
span = active_span
|
|
87
69
|
|
|
88
70
|
Rack::Reactive::Response.subscribe(op, waf_context) do |action, result, _block|
|
|
89
71
|
record = [:block, :monitor].include?(action)
|
|
@@ -91,11 +73,13 @@ module Datadog
|
|
|
91
73
|
# TODO: should this hash be an Event instance instead?
|
|
92
74
|
event = {
|
|
93
75
|
waf_result: result,
|
|
94
|
-
trace:
|
|
95
|
-
span:
|
|
76
|
+
trace: trace,
|
|
77
|
+
span: span,
|
|
96
78
|
response: response,
|
|
97
79
|
action: action
|
|
98
80
|
}
|
|
81
|
+
|
|
82
|
+
waf_context.events << event
|
|
99
83
|
end
|
|
100
84
|
end
|
|
101
85
|
|
|
@@ -113,12 +97,71 @@ module Datadog
|
|
|
113
97
|
|
|
114
98
|
[ret, res]
|
|
115
99
|
end
|
|
100
|
+
|
|
101
|
+
Instrumentation.gateway.watch('rack.request.body') do |stack, request|
|
|
102
|
+
block = false
|
|
103
|
+
event = nil
|
|
104
|
+
waf_context = request.env['datadog.waf.context']
|
|
105
|
+
|
|
106
|
+
AppSec::Reactive::Operation.new('rack.request.body') do |op|
|
|
107
|
+
trace = active_trace
|
|
108
|
+
span = active_span
|
|
109
|
+
|
|
110
|
+
Rack::Reactive::RequestBody.subscribe(op, waf_context) do |action, result, _block|
|
|
111
|
+
record = [:block, :monitor].include?(action)
|
|
112
|
+
if record
|
|
113
|
+
# TODO: should this hash be an Event instance instead?
|
|
114
|
+
event = {
|
|
115
|
+
waf_result: result,
|
|
116
|
+
trace: trace,
|
|
117
|
+
span: span,
|
|
118
|
+
request: request,
|
|
119
|
+
action: action
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
waf_context.events << event
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
_action, _result, block = Rack::Reactive::RequestBody.publish(op, request)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
next [nil, [[:block, event]]] if block
|
|
130
|
+
|
|
131
|
+
ret, res = stack.call(request)
|
|
132
|
+
|
|
133
|
+
if event
|
|
134
|
+
res ||= []
|
|
135
|
+
res << [:monitor, event]
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
[ret, res]
|
|
139
|
+
end
|
|
116
140
|
end
|
|
117
|
-
# rubocop:enable Metrics/AbcSize
|
|
118
|
-
# rubocop:enable Metrics/CyclomaticComplexity
|
|
119
141
|
# rubocop:enable Metrics/MethodLength
|
|
120
|
-
# rubocop:enable Metrics/
|
|
142
|
+
# rubocop:enable Metrics/AbcSize
|
|
143
|
+
|
|
144
|
+
class << self
|
|
145
|
+
private
|
|
146
|
+
|
|
147
|
+
def active_trace
|
|
148
|
+
# TODO: factor out tracing availability detection
|
|
149
|
+
|
|
150
|
+
return unless defined?(Datadog::Tracing)
|
|
151
|
+
|
|
152
|
+
Datadog::Tracing.active_trace
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def active_span
|
|
156
|
+
# TODO: factor out tracing availability detection
|
|
157
|
+
|
|
158
|
+
return unless defined?(Datadog::Tracing)
|
|
159
|
+
|
|
160
|
+
Datadog::Tracing.active_span
|
|
161
|
+
end
|
|
162
|
+
end
|
|
121
163
|
end
|
|
164
|
+
# rubocop:enable Metrics/ModuleLength
|
|
122
165
|
end
|
|
123
166
|
end
|
|
124
167
|
end
|
|
@@ -5,6 +5,7 @@ require 'datadog/appsec/contrib/integration'
|
|
|
5
5
|
require 'datadog/appsec/contrib/rack/configuration/settings'
|
|
6
6
|
require 'datadog/appsec/contrib/rack/patcher'
|
|
7
7
|
require 'datadog/appsec/contrib/rack/request_middleware'
|
|
8
|
+
require 'datadog/appsec/contrib/rack/request_body_middleware'
|
|
8
9
|
|
|
9
10
|
module Datadog
|
|
10
11
|
module AppSec
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# typed: true
|
|
2
|
+
|
|
3
|
+
require 'datadog/appsec/contrib/rack/request'
|
|
4
|
+
|
|
5
|
+
module Datadog
|
|
6
|
+
module AppSec
|
|
7
|
+
module Contrib
|
|
8
|
+
module Rack
|
|
9
|
+
module Reactive
|
|
10
|
+
# Dispatch data from a Rack request to the WAF context
|
|
11
|
+
module RequestBody
|
|
12
|
+
def self.publish(op, request)
|
|
13
|
+
catch(:block) do
|
|
14
|
+
# params have been parsed from the request body
|
|
15
|
+
op.publish('request.body', Rack::Request.form_hash(request))
|
|
16
|
+
|
|
17
|
+
nil
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.subscribe(op, waf_context)
|
|
22
|
+
addresses = [
|
|
23
|
+
'request.body',
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
op.subscribe(*addresses) do |*values|
|
|
27
|
+
Datadog.logger.debug { "reacted to #{addresses.inspect}: #{values.inspect}" }
|
|
28
|
+
body = values[0]
|
|
29
|
+
|
|
30
|
+
waf_args = {
|
|
31
|
+
'server.request.body' => body,
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
waf_timeout = Datadog::AppSec.settings.waf_timeout
|
|
35
|
+
action, result = waf_context.run(waf_args, waf_timeout)
|
|
36
|
+
|
|
37
|
+
Datadog.logger.debug { "WAF TIMEOUT: #{result.inspect}" } if result.timeout
|
|
38
|
+
|
|
39
|
+
# TODO: encapsulate return array in a type
|
|
40
|
+
case action
|
|
41
|
+
when :monitor
|
|
42
|
+
Datadog.logger.debug { "WAF: #{result.inspect}" }
|
|
43
|
+
yield [action, result, false]
|
|
44
|
+
when :block
|
|
45
|
+
Datadog.logger.debug { "WAF: #{result.inspect}" }
|
|
46
|
+
yield [action, result, true]
|
|
47
|
+
throw(:block, [action, result, true])
|
|
48
|
+
when :good
|
|
49
|
+
Datadog.logger.debug { "WAF OK: #{result.inspect}" }
|
|
50
|
+
when :invalid_call
|
|
51
|
+
Datadog.logger.debug { "WAF CALL ERROR: #{result.inspect}" }
|
|
52
|
+
when :invalid_rule, :invalid_flow, :no_rule
|
|
53
|
+
Datadog.logger.debug { "WAF RULE ERROR: #{result.inspect}" }
|
|
54
|
+
else
|
|
55
|
+
Datadog.logger.debug { "WAF UNKNOWN: #{action.inspect} #{result.inspect}" }
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -45,6 +45,12 @@ module Datadog
|
|
|
45
45
|
def self.cookies(request)
|
|
46
46
|
request.cookies
|
|
47
47
|
end
|
|
48
|
+
|
|
49
|
+
def self.form_hash(request)
|
|
50
|
+
# usually Hash<String,String> but can be a more complex
|
|
51
|
+
# Hash<String,String||Array||Hash> when e.g coming from JSON
|
|
52
|
+
request.env['rack.request.form_hash']
|
|
53
|
+
end
|
|
48
54
|
end
|
|
49
55
|
end
|
|
50
56
|
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# typed: ignore
|
|
2
|
+
|
|
3
|
+
require 'datadog/appsec/instrumentation/gateway'
|
|
4
|
+
require 'datadog/appsec/assets'
|
|
5
|
+
|
|
6
|
+
module Datadog
|
|
7
|
+
module AppSec
|
|
8
|
+
module Contrib
|
|
9
|
+
module Rack
|
|
10
|
+
# Rack request body middleware for AppSec
|
|
11
|
+
# This should be inserted just below Rack::JSONBodyParser or
|
|
12
|
+
# legacy Rack::PostBodyContentTypeParser from rack-contrib
|
|
13
|
+
class RequestBodyMiddleware
|
|
14
|
+
def initialize(app, opt = {})
|
|
15
|
+
@app = app
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def call(env)
|
|
19
|
+
context = env['datadog.waf.context']
|
|
20
|
+
|
|
21
|
+
return @app.call(env) unless context
|
|
22
|
+
|
|
23
|
+
# TODO: handle exceptions, except for @app.call
|
|
24
|
+
|
|
25
|
+
request = ::Rack::Request.new(env)
|
|
26
|
+
|
|
27
|
+
request_return, request_response = Instrumentation.gateway.push('rack.request.body', request) do
|
|
28
|
+
@app.call(env)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
if request_response && request_response.any? { |action, _event| action == :block }
|
|
32
|
+
request_return = [403, { 'Content-Type' => 'text/html' }, [Datadog::AppSec::Assets.blocked]]
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
request_return
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# typed: ignore
|
|
2
2
|
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
3
5
|
require 'datadog/appsec/instrumentation/gateway'
|
|
4
6
|
require 'datadog/appsec/processor'
|
|
5
7
|
require 'datadog/appsec/assets'
|
|
@@ -14,6 +16,7 @@ module Datadog
|
|
|
14
16
|
def initialize(app, opt = {})
|
|
15
17
|
@app = app
|
|
16
18
|
|
|
19
|
+
@oneshot_tags_sent = false
|
|
17
20
|
@processor = Datadog::AppSec::Processor.new
|
|
18
21
|
end
|
|
19
22
|
|
|
@@ -27,6 +30,8 @@ module Datadog
|
|
|
27
30
|
env['datadog.waf.context'] = context
|
|
28
31
|
request = ::Rack::Request.new(env)
|
|
29
32
|
|
|
33
|
+
add_appsec_tags
|
|
34
|
+
|
|
30
35
|
request_return, request_response = Instrumentation.gateway.push('rack.request', request) do
|
|
31
36
|
@app.call(env)
|
|
32
37
|
end
|
|
@@ -40,15 +45,65 @@ module Datadog
|
|
|
40
45
|
@waf_context = context
|
|
41
46
|
end
|
|
42
47
|
|
|
43
|
-
|
|
48
|
+
_response_return, _response_response = Instrumentation.gateway.push('rack.response', response)
|
|
44
49
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
50
|
+
context.events.each do |e|
|
|
51
|
+
e[:response] ||= response
|
|
52
|
+
e[:request] ||= request
|
|
53
|
+
end
|
|
48
54
|
|
|
49
|
-
AppSec::Event.record(*
|
|
55
|
+
AppSec::Event.record(*context.events)
|
|
50
56
|
|
|
51
57
|
request_return
|
|
58
|
+
ensure
|
|
59
|
+
add_waf_runtime_tags(context) if context
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
def active_trace
|
|
65
|
+
# TODO: factor out tracing availability detection
|
|
66
|
+
|
|
67
|
+
return unless defined?(Datadog::Tracing)
|
|
68
|
+
|
|
69
|
+
Datadog::Tracing.active_trace
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def add_appsec_tags
|
|
73
|
+
return unless active_trace
|
|
74
|
+
|
|
75
|
+
active_trace.set_tag('_dd.appsec.enabled', 1)
|
|
76
|
+
active_trace.set_tag('_dd.runtime_family', 'ruby')
|
|
77
|
+
active_trace.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
|
|
78
|
+
|
|
79
|
+
if @processor.ruleset_info
|
|
80
|
+
active_trace.set_tag('_dd.appsec.event_rules.version', @processor.ruleset_info[:version])
|
|
81
|
+
|
|
82
|
+
unless @oneshot_tags_sent
|
|
83
|
+
# Small race condition, but it's inoccuous: worst case the tags
|
|
84
|
+
# are sent a couple of times more than expected
|
|
85
|
+
@oneshot_tags_sent = true
|
|
86
|
+
|
|
87
|
+
active_trace.set_tag('_dd.appsec.event_rules.loaded', @processor.ruleset_info[:loaded].to_f)
|
|
88
|
+
active_trace.set_tag('_dd.appsec.event_rules.error_count', @processor.ruleset_info[:failed].to_f)
|
|
89
|
+
active_trace.set_tag('_dd.appsec.event_rules.errors', JSON.dump(@processor.ruleset_info[:errors]))
|
|
90
|
+
active_trace.set_tag('_dd.appsec.event_rules.addresses', JSON.dump(@processor.addresses))
|
|
91
|
+
|
|
92
|
+
# Ensure these tags reach the backend
|
|
93
|
+
active_trace.keep!
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def add_waf_runtime_tags(context)
|
|
99
|
+
return unless active_trace
|
|
100
|
+
return unless context
|
|
101
|
+
|
|
102
|
+
active_trace.set_tag('_dd.appsec.waf.timeouts', context.timeouts)
|
|
103
|
+
|
|
104
|
+
# these tags expect time in us
|
|
105
|
+
active_trace.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
|
|
106
|
+
active_trace.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
|
|
52
107
|
end
|
|
53
108
|
end
|
|
54
109
|
end
|