ddtrace 1.17.0 → 1.19.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (54) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +85 -2
  3. data/ext/ddtrace_profiling_native_extension/clock_id_from_pthread.c +3 -0
  4. data/ext/ddtrace_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +67 -52
  5. data/ext/ddtrace_profiling_native_extension/collectors_dynamic_sampling_rate.c +22 -14
  6. data/ext/ddtrace_profiling_native_extension/collectors_dynamic_sampling_rate.h +4 -0
  7. data/ext/ddtrace_profiling_native_extension/collectors_gc_profiling_helper.c +156 -0
  8. data/ext/ddtrace_profiling_native_extension/collectors_gc_profiling_helper.h +5 -0
  9. data/ext/ddtrace_profiling_native_extension/collectors_stack.c +43 -102
  10. data/ext/ddtrace_profiling_native_extension/collectors_stack.h +10 -3
  11. data/ext/ddtrace_profiling_native_extension/collectors_thread_context.c +167 -125
  12. data/ext/ddtrace_profiling_native_extension/collectors_thread_context.h +2 -1
  13. data/ext/ddtrace_profiling_native_extension/extconf.rb +44 -10
  14. data/ext/ddtrace_profiling_native_extension/heap_recorder.c +970 -0
  15. data/ext/ddtrace_profiling_native_extension/heap_recorder.h +155 -0
  16. data/ext/ddtrace_profiling_native_extension/helpers.h +2 -0
  17. data/ext/ddtrace_profiling_native_extension/http_transport.c +5 -2
  18. data/ext/ddtrace_profiling_native_extension/libdatadog_helpers.c +20 -0
  19. data/ext/ddtrace_profiling_native_extension/libdatadog_helpers.h +11 -0
  20. data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c +83 -18
  21. data/ext/ddtrace_profiling_native_extension/private_vm_api_access.h +6 -0
  22. data/ext/ddtrace_profiling_native_extension/profiling.c +2 -0
  23. data/ext/ddtrace_profiling_native_extension/ruby_helpers.c +147 -0
  24. data/ext/ddtrace_profiling_native_extension/ruby_helpers.h +28 -0
  25. data/ext/ddtrace_profiling_native_extension/stack_recorder.c +330 -13
  26. data/ext/ddtrace_profiling_native_extension/stack_recorder.h +3 -0
  27. data/lib/datadog/appsec/component.rb +4 -1
  28. data/lib/datadog/appsec/configuration/settings.rb +4 -0
  29. data/lib/datadog/appsec/contrib/devise/patcher/registration_controller_patch.rb +2 -0
  30. data/lib/datadog/appsec/processor/rule_loader.rb +60 -0
  31. data/lib/datadog/appsec/remote.rb +12 -9
  32. data/lib/datadog/core/configuration/settings.rb +139 -22
  33. data/lib/datadog/core/configuration.rb +4 -0
  34. data/lib/datadog/core/remote/worker.rb +1 -0
  35. data/lib/datadog/core/telemetry/collector.rb +10 -0
  36. data/lib/datadog/core/telemetry/event.rb +2 -1
  37. data/lib/datadog/core/telemetry/ext.rb +3 -0
  38. data/lib/datadog/core/telemetry/v1/app_event.rb +8 -1
  39. data/lib/datadog/core/telemetry/v1/install_signature.rb +38 -0
  40. data/lib/datadog/core/workers/async.rb +1 -0
  41. data/lib/datadog/kit/enable_core_dumps.rb +5 -6
  42. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +7 -11
  43. data/lib/datadog/profiling/collectors/idle_sampling_helper.rb +1 -0
  44. data/lib/datadog/profiling/component.rb +210 -18
  45. data/lib/datadog/profiling/scheduler.rb +4 -6
  46. data/lib/datadog/profiling/stack_recorder.rb +13 -2
  47. data/lib/datadog/tracing/contrib/mysql2/configuration/settings.rb +4 -0
  48. data/lib/datadog/tracing/contrib/mysql2/instrumentation.rb +2 -1
  49. data/lib/datadog/tracing/contrib/pg/configuration/settings.rb +5 -0
  50. data/lib/datadog/tracing/contrib/pg/instrumentation.rb +24 -0
  51. data/lib/datadog/tracing/contrib/rails/auto_instrument_railtie.rb +0 -2
  52. data/lib/datadog/tracing/workers.rb +1 -0
  53. data/lib/ddtrace/version.rb +1 -1
  54. metadata +11 -6
@@ -0,0 +1,5 @@
1
+ #pragma once
2
+
3
+ void gc_profiling_init(void);
4
+ bool gc_profiling_has_major_gc_finished(void);
5
+ uint8_t gc_profiling_set_metadata(ddog_prof_Label *labels, int labels_length);
@@ -11,9 +11,6 @@
11
11
  // Gathers stack traces from running threads, storing them in a StackRecorder instance
12
12
  // This file implements the native bits of the Datadog::Profiling::Collectors::Stack class
13
13
 
14
- #define MAX_FRAMES_LIMIT 10000
15
- #define MAX_FRAMES_LIMIT_AS_STRING "10000"
16
-
17
14
  static VALUE missing_string = Qnil;
18
15
 
19
16
  // Used as scratch space during sampling
@@ -36,23 +33,7 @@ static VALUE _native_sample(
36
33
  VALUE in_gc
37
34
  );
38
35
  static void maybe_add_placeholder_frames_omitted(VALUE thread, sampling_buffer* buffer, char *frames_omitted_message, int frames_omitted_message_size);
39
- static void record_placeholder_stack_in_native_code(
40
- sampling_buffer* buffer,
41
- VALUE recorder_instance,
42
- sample_values values,
43
- sample_labels labels,
44
- sampling_buffer *record_buffer,
45
- int extra_frames_in_record_buffer
46
- );
47
- static void sample_thread_internal(
48
- VALUE thread,
49
- sampling_buffer* buffer,
50
- VALUE recorder_instance,
51
- sample_values values,
52
- sample_labels labels,
53
- sampling_buffer *record_buffer,
54
- int extra_frames_in_record_buffer
55
- );
36
+ static void record_placeholder_stack_in_native_code(sampling_buffer* buffer, VALUE recorder_instance, sample_values values, sample_labels labels);
56
37
 
57
38
  void collectors_stack_init(VALUE profiling_module) {
58
39
  VALUE collectors_module = rb_define_module_under(profiling_module, "Collectors");
@@ -88,6 +69,7 @@ static VALUE _native_sample(
88
69
  .cpu_or_wall_samples = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("cpu-samples"), zero)),
89
70
  .wall_time_ns = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("wall-time"), zero)),
90
71
  .alloc_samples = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("alloc-samples"), zero)),
72
+ .timeline_wall_time_ns = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("timeline"), zero)),
91
73
  };
92
74
 
93
75
  long labels_count = RARRAY_LEN(labels_array) + RARRAY_LEN(numeric_labels_array);
@@ -122,62 +104,29 @@ static VALUE _native_sample(
122
104
 
123
105
  ddog_prof_Slice_Label slice_labels = {.ptr = labels, .len = labels_count};
124
106
 
125
- sample_thread(
126
- thread,
127
- buffer,
128
- recorder_instance,
129
- values,
130
- (sample_labels) {.labels = slice_labels, .state_label = state_label},
131
- RTEST(in_gc) ? SAMPLE_IN_GC : SAMPLE_REGULAR
132
- );
107
+ if (in_gc == Qtrue) {
108
+ record_placeholder_stack(
109
+ buffer,
110
+ recorder_instance,
111
+ values,
112
+ (sample_labels) {.labels = slice_labels, .state_label = state_label},
113
+ DDOG_CHARSLICE_C("Garbage Collection")
114
+ );
115
+ } else {
116
+ sample_thread(
117
+ thread,
118
+ buffer,
119
+ recorder_instance,
120
+ values,
121
+ (sample_labels) {.labels = slice_labels, .state_label = state_label}
122
+ );
123
+ }
133
124
 
134
125
  sampling_buffer_free(buffer);
135
126
 
136
127
  return Qtrue;
137
128
  }
138
129
 
139
- void sample_thread(
140
- VALUE thread,
141
- sampling_buffer* buffer,
142
- VALUE recorder_instance,
143
- sample_values values,
144
- sample_labels labels,
145
- sample_type type
146
- ) {
147
- // Samples thread into recorder
148
- if (type == SAMPLE_REGULAR) {
149
- sampling_buffer *record_buffer = buffer;
150
- int extra_frames_in_record_buffer = 0;
151
- sample_thread_internal(thread, buffer, recorder_instance, values, labels, record_buffer, extra_frames_in_record_buffer);
152
- return;
153
- }
154
-
155
- // Samples thread into recorder, including as a top frame in the stack a frame named "Garbage Collection"
156
- if (type == SAMPLE_IN_GC) {
157
- ddog_CharSlice function_name = DDOG_CHARSLICE_C("");
158
- ddog_CharSlice function_filename = DDOG_CHARSLICE_C("Garbage Collection");
159
- buffer->locations[0] = (ddog_prof_Location) {
160
- .function = (ddog_prof_Function) {.name = function_name, .filename = function_filename},
161
- .line = 0
162
- };
163
- // To avoid changing sample_thread_internal, we just prepare a new buffer struct that uses the same underlying storage as the
164
- // original buffer, but has capacity one less, so that we can keep the above Garbage Collection frame untouched.
165
- sampling_buffer thread_in_gc_buffer = (struct sampling_buffer) {
166
- .max_frames = buffer->max_frames - 1,
167
- .stack_buffer = buffer->stack_buffer + 1,
168
- .lines_buffer = buffer->lines_buffer + 1,
169
- .is_ruby_frame = buffer->is_ruby_frame + 1,
170
- .locations = buffer->locations + 1,
171
- };
172
- sampling_buffer *record_buffer = buffer; // We pass in the original buffer as the record_buffer, but not as the regular buffer
173
- int extra_frames_in_record_buffer = 1;
174
- sample_thread_internal(thread, &thread_in_gc_buffer, recorder_instance, values, labels, record_buffer, extra_frames_in_record_buffer);
175
- return;
176
- }
177
-
178
- rb_raise(rb_eArgError, "Unexpected value for sample_type: %d", type);
179
- }
180
-
181
130
  #define CHARSLICE_EQUALS(must_be_a_literal, charslice) (strlen("" must_be_a_literal) == charslice.len && strncmp(must_be_a_literal, charslice.ptr, charslice.len) == 0)
182
131
 
183
132
  // Idea: Should we release the global vm lock (GVL) after we get the data from `rb_profile_frames`? That way other Ruby threads
@@ -189,24 +138,12 @@ void sample_thread(
189
138
  // * Should we move this into a different thread entirely?
190
139
  // * If we don't move it into a different thread, does releasing the GVL on a Ruby thread mean that we're introducing
191
140
  // a new thread switch point where there previously was none?
192
- //
193
- // ---
194
- //
195
- // Why the weird extra record_buffer and extra_frames_in_record_buffer?
196
- // The answer is: to support both sample_thread() and sample_thread_in_gc().
197
- //
198
- // For sample_thread(), buffer == record_buffer and extra_frames_in_record_buffer == 0, so it's a no-op.
199
- // For sample_thread_in_gc(), the buffer is a special buffer that is the same as the record_buffer, but with every
200
- // pointer shifted forward extra_frames_in_record_buffer elements, so that the caller can actually inject those extra
201
- // frames, and this function doesn't have to care about it.
202
- static void sample_thread_internal(
141
+ void sample_thread(
203
142
  VALUE thread,
204
143
  sampling_buffer* buffer,
205
144
  VALUE recorder_instance,
206
145
  sample_values values,
207
- sample_labels labels,
208
- sampling_buffer *record_buffer,
209
- int extra_frames_in_record_buffer
146
+ sample_labels labels
210
147
  ) {
211
148
  int captured_frames = ddtrace_rb_profile_frames(
212
149
  thread,
@@ -218,14 +155,7 @@ static void sample_thread_internal(
218
155
  );
219
156
 
220
157
  if (captured_frames == PLACEHOLDER_STACK_IN_NATIVE_CODE) {
221
- record_placeholder_stack_in_native_code(
222
- buffer,
223
- recorder_instance,
224
- values,
225
- labels,
226
- record_buffer,
227
- extra_frames_in_record_buffer
228
- );
158
+ record_placeholder_stack_in_native_code(buffer, recorder_instance, values, labels);
229
159
  return;
230
160
  }
231
161
 
@@ -332,7 +262,7 @@ static void sample_thread_internal(
332
262
 
333
263
  record_sample(
334
264
  recorder_instance,
335
- (ddog_prof_Slice_Location) {.ptr = record_buffer->locations, .len = captured_frames + extra_frames_in_record_buffer},
265
+ (ddog_prof_Slice_Location) {.ptr = buffer->locations, .len = captured_frames},
336
266
  values,
337
267
  labels
338
268
  );
@@ -379,24 +309,35 @@ static void maybe_add_placeholder_frames_omitted(VALUE thread, sampling_buffer*
379
309
  //
380
310
  // To give customers visibility into these threads, rather than reporting an empty stack, we replace the empty stack
381
311
  // with one containing a placeholder frame, so that these threads are properly represented in the UX.
312
+
382
313
  static void record_placeholder_stack_in_native_code(
314
+ sampling_buffer* buffer,
315
+ VALUE recorder_instance,
316
+ sample_values values,
317
+ sample_labels labels
318
+ ) {
319
+ record_placeholder_stack(
320
+ buffer,
321
+ recorder_instance,
322
+ values,
323
+ labels,
324
+ DDOG_CHARSLICE_C("In native code")
325
+ );
326
+ }
327
+
328
+ void record_placeholder_stack(
383
329
  sampling_buffer* buffer,
384
330
  VALUE recorder_instance,
385
331
  sample_values values,
386
332
  sample_labels labels,
387
- sampling_buffer *record_buffer,
388
- int extra_frames_in_record_buffer
333
+ ddog_CharSlice placeholder_stack
389
334
  ) {
390
- ddog_CharSlice function_name = DDOG_CHARSLICE_C("");
391
- ddog_CharSlice function_filename = DDOG_CHARSLICE_C("In native code");
392
- buffer->locations[0] = (ddog_prof_Location) {
393
- .function = (ddog_prof_Function) {.name = function_name, .filename = function_filename},
394
- .line = 0
395
- };
335
+ ddog_prof_Function placeholder = {.name = DDOG_CHARSLICE_C(""), .filename = placeholder_stack};
336
+ buffer->locations[0] = (ddog_prof_Location) {.function = placeholder, .line = 0};
396
337
 
397
338
  record_sample(
398
339
  recorder_instance,
399
- (ddog_prof_Slice_Location) {.ptr = record_buffer->locations, .len = 1 + extra_frames_in_record_buffer},
340
+ (ddog_prof_Slice_Location) {.ptr = buffer->locations, .len = 1},
400
341
  values,
401
342
  labels
402
343
  );
@@ -4,17 +4,24 @@
4
4
 
5
5
  #include "stack_recorder.h"
6
6
 
7
- typedef struct sampling_buffer sampling_buffer;
7
+ #define MAX_FRAMES_LIMIT 10000
8
+ #define MAX_FRAMES_LIMIT_AS_STRING "10000"
8
9
 
9
- typedef enum { SAMPLE_REGULAR, SAMPLE_IN_GC } sample_type;
10
+ typedef struct sampling_buffer sampling_buffer;
10
11
 
11
12
  void sample_thread(
12
13
  VALUE thread,
14
+ sampling_buffer* buffer,
15
+ VALUE recorder_instance,
16
+ sample_values values,
17
+ sample_labels labels
18
+ );
19
+ void record_placeholder_stack(
13
20
  sampling_buffer* buffer,
14
21
  VALUE recorder_instance,
15
22
  sample_values values,
16
23
  sample_labels labels,
17
- sample_type type
24
+ ddog_CharSlice placeholder_stack
18
25
  );
19
26
  sampling_buffer *sampling_buffer_new(unsigned int max_frames);
20
27
  void sampling_buffer_free(sampling_buffer *buffer);