ddtrace 1.6.0 → 1.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +50 -1
  3. data/ext/ddtrace_profiling_loader/extconf.rb +1 -1
  4. data/ext/ddtrace_profiling_native_extension/collectors_cpu_and_wall_time.c +66 -6
  5. data/ext/ddtrace_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +51 -54
  6. data/ext/ddtrace_profiling_native_extension/collectors_stack.c +11 -13
  7. data/ext/ddtrace_profiling_native_extension/extconf.rb +1 -1
  8. data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c +3 -2
  9. data/ext/ddtrace_profiling_native_extension/setup_signal_handler.c +96 -0
  10. data/ext/ddtrace_profiling_native_extension/setup_signal_handler.h +7 -0
  11. data/ext/ddtrace_profiling_native_extension/stack_recorder.c +70 -18
  12. data/ext/ddtrace_profiling_native_extension/stack_recorder.h +1 -0
  13. data/lib/datadog/appsec/assets/blocked.html +98 -3
  14. data/lib/datadog/appsec/assets/blocked.json +1 -0
  15. data/lib/datadog/appsec/assets/blocked.text +5 -0
  16. data/lib/datadog/appsec/assets/waf_rules/recommended.json +35 -46
  17. data/lib/datadog/appsec/assets/waf_rules/risky.json +1 -1
  18. data/lib/datadog/appsec/assets/waf_rules/strict.json +46 -1
  19. data/lib/datadog/appsec/assets.rb +2 -2
  20. data/lib/datadog/appsec/configuration/settings.rb +6 -0
  21. data/lib/datadog/appsec/configuration.rb +4 -0
  22. data/lib/datadog/appsec/contrib/rack/reactive/request.rb +4 -8
  23. data/lib/datadog/appsec/contrib/rack/request.rb +17 -0
  24. data/lib/datadog/appsec/contrib/rack/request_body_middleware.rb +2 -2
  25. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +2 -2
  26. data/lib/datadog/appsec/contrib/rails/patcher.rb +3 -6
  27. data/lib/datadog/appsec/contrib/sinatra/ext.rb +1 -0
  28. data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb +1 -1
  29. data/lib/datadog/appsec/contrib/sinatra/patcher.rb +11 -8
  30. data/lib/datadog/appsec/extensions.rb +10 -0
  31. data/lib/datadog/appsec/processor.rb +18 -0
  32. data/lib/datadog/appsec/response.rb +54 -0
  33. data/lib/datadog/core/runtime/ext.rb +1 -1
  34. data/lib/datadog/opentracer/distributed_headers.rb +5 -7
  35. data/lib/datadog/opentracer/rack_propagator.rb +0 -3
  36. data/lib/datadog/opentracer/text_map_propagator.rb +5 -7
  37. data/lib/datadog/profiling/collectors/cpu_and_wall_time.rb +10 -4
  38. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +4 -0
  39. data/lib/datadog/profiling/collectors/old_stack.rb +7 -0
  40. data/lib/datadog/profiling/exporter.rb +5 -0
  41. data/lib/datadog/profiling/old_recorder.rb +8 -0
  42. data/lib/datadog/profiling/profiler.rb +7 -0
  43. data/lib/datadog/profiling/scheduler.rb +4 -7
  44. data/lib/datadog/profiling/stack_recorder.rb +22 -0
  45. data/lib/datadog/profiling/tasks/setup.rb +0 -7
  46. data/lib/datadog/tracing/contrib/delayed_job/plugin.rb +4 -0
  47. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/client.rb +2 -1
  48. data/lib/datadog/tracing/contrib/grpc/datadog_interceptor/server.rb +6 -12
  49. data/lib/datadog/tracing/contrib/grpc/distributed/fetcher.rb +27 -0
  50. data/lib/datadog/tracing/contrib/grpc/distributed/propagation.rb +38 -0
  51. data/lib/datadog/tracing/contrib/grpc/patcher.rb +0 -2
  52. data/lib/datadog/tracing/contrib/http/distributed/fetcher.rb +32 -0
  53. data/lib/datadog/tracing/contrib/http/distributed/propagation.rb +33 -0
  54. data/lib/datadog/tracing/contrib/kafka/consumer_event.rb +1 -0
  55. data/lib/datadog/tracing/contrib/kafka/events/produce_operation/send_messages.rb +1 -0
  56. data/lib/datadog/tracing/contrib/kafka/events/producer/deliver_messages.rb +1 -0
  57. data/lib/datadog/tracing/contrib/mongodb/subscribers.rb +2 -0
  58. data/lib/datadog/tracing/contrib/que/tracer.rb +2 -0
  59. data/lib/datadog/tracing/contrib/racecar/events/batch.rb +4 -1
  60. data/lib/datadog/tracing/contrib/racecar/events/message.rb +4 -1
  61. data/lib/datadog/tracing/contrib/rack/middlewares.rb +2 -0
  62. data/lib/datadog/tracing/contrib/redis/instrumentation.rb +2 -0
  63. data/lib/datadog/tracing/contrib/redis/integration.rb +2 -1
  64. data/lib/datadog/tracing/contrib/redis/patcher.rb +2 -3
  65. data/lib/datadog/tracing/contrib/resque/resque_job.rb +2 -0
  66. data/lib/datadog/tracing/contrib/shoryuken/tracer.rb +2 -0
  67. data/lib/datadog/tracing/contrib/sidekiq/client_tracer.rb +5 -0
  68. data/lib/datadog/tracing/contrib/sidekiq/server_tracer.rb +5 -0
  69. data/lib/datadog/tracing/contrib/sneakers/tracer.rb +2 -0
  70. data/lib/datadog/tracing/distributed/b3.rb +66 -0
  71. data/lib/datadog/tracing/distributed/b3_single.rb +66 -0
  72. data/lib/datadog/tracing/distributed/datadog.rb +153 -0
  73. data/lib/datadog/tracing/distributed/datadog_tags_codec.rb +1 -0
  74. data/lib/datadog/tracing/distributed/fetcher.rb +30 -0
  75. data/lib/datadog/tracing/distributed/headers/ext.rb +18 -16
  76. data/lib/datadog/tracing/distributed/helpers.rb +7 -6
  77. data/lib/datadog/tracing/distributed/propagation.rb +127 -0
  78. data/lib/datadog/tracing/propagation/http.rb +3 -106
  79. data/lib/datadog/tracing/trace_segment.rb +1 -1
  80. data/lib/ddtrace/transport/trace_formatter.rb +2 -5
  81. data/lib/ddtrace/version.rb +1 -1
  82. metadata +19 -14
  83. data/lib/datadog/tracing/distributed/headers/b3.rb +0 -55
  84. data/lib/datadog/tracing/distributed/headers/b3_single.rb +0 -67
  85. data/lib/datadog/tracing/distributed/headers/datadog.rb +0 -144
  86. data/lib/datadog/tracing/distributed/headers/parser.rb +0 -37
  87. data/lib/datadog/tracing/distributed/metadata/b3.rb +0 -55
  88. data/lib/datadog/tracing/distributed/metadata/b3_single.rb +0 -66
  89. data/lib/datadog/tracing/distributed/metadata/datadog.rb +0 -73
  90. data/lib/datadog/tracing/distributed/metadata/parser.rb +0 -34
  91. data/lib/datadog/tracing/propagation/grpc.rb +0 -98
@@ -152,6 +152,7 @@ struct active_slot_pair {
152
152
  struct call_serialize_without_gvl_arguments {
153
153
  // Set by caller
154
154
  struct stack_recorder_state *state;
155
+ ddog_Timespec finish_timestamp;
155
156
 
156
157
  // Set by callee
157
158
  ddog_Profile *profile;
@@ -162,18 +163,22 @@ struct call_serialize_without_gvl_arguments {
162
163
  };
163
164
 
164
165
  static VALUE _native_new(VALUE klass);
166
+ static void initialize_slot_concurrency_control(struct stack_recorder_state *state);
165
167
  static void stack_recorder_typed_data_free(void *data);
166
168
  static VALUE _native_serialize(VALUE self, VALUE recorder_instance);
167
169
  static VALUE ruby_time_from(ddog_Timespec ddprof_time);
168
170
  static void *call_serialize_without_gvl(void *call_args);
169
171
  static struct active_slot_pair sampler_lock_active_profile();
170
172
  static void sampler_unlock_active_profile(struct active_slot_pair active_slot);
171
- static ddog_Profile *serializer_flip_active_and_inactive_slots(struct stack_recorder_state *state, ddog_Timespec start_timestamp_for_next_profile);
173
+ static ddog_Profile *serializer_flip_active_and_inactive_slots(struct stack_recorder_state *state);
172
174
  static VALUE _native_active_slot(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance);
173
175
  static VALUE _native_is_slot_one_mutex_locked(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance);
174
176
  static VALUE _native_is_slot_two_mutex_locked(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance);
175
177
  static VALUE test_slot_mutex_state(VALUE recorder_instance, int slot);
176
178
  static ddog_Timespec time_now();
179
+ static VALUE _native_reset_after_fork(DDTRACE_UNUSED VALUE self, VALUE recorder_instance);
180
+ static void serializer_set_start_timestamp_for_next_profile(struct stack_recorder_state *state, ddog_Timespec timestamp);
181
+ static VALUE _native_record_endpoint(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance, VALUE local_root_span_id, VALUE endpoint);
177
182
 
178
183
  void stack_recorder_init(VALUE profiling_module) {
179
184
  stack_recorder_class = rb_define_class_under(profiling_module, "StackRecorder", rb_cObject);
@@ -191,9 +196,11 @@ void stack_recorder_init(VALUE profiling_module) {
191
196
  rb_define_alloc_func(stack_recorder_class, _native_new);
192
197
 
193
198
  rb_define_singleton_method(stack_recorder_class, "_native_serialize", _native_serialize, 1);
199
+ rb_define_singleton_method(stack_recorder_class, "_native_reset_after_fork", _native_reset_after_fork, 1);
194
200
  rb_define_singleton_method(testing_module, "_native_active_slot", _native_active_slot, 1);
195
201
  rb_define_singleton_method(testing_module, "_native_slot_one_mutex_locked?", _native_is_slot_one_mutex_locked, 1);
196
202
  rb_define_singleton_method(testing_module, "_native_slot_two_mutex_locked?", _native_is_slot_two_mutex_locked, 1);
203
+ rb_define_singleton_method(testing_module, "_native_record_endpoint", _native_record_endpoint, 3);
197
204
 
198
205
  ok_symbol = ID2SYM(rb_intern_const("ok"));
199
206
  error_symbol = ID2SYM(rb_intern_const("error"));
@@ -217,6 +224,17 @@ static VALUE _native_new(VALUE klass) {
217
224
 
218
225
  ddog_Slice_value_type sample_types = {.ptr = enabled_value_types, .len = ENABLED_VALUE_TYPES_COUNT};
219
226
 
227
+ initialize_slot_concurrency_control(state);
228
+
229
+ // Note: Don't raise exceptions after this point, since it'll lead to libdatadog memory leaking!
230
+
231
+ state->slot_one_profile = ddog_Profile_new(sample_types, NULL /* period is optional */, NULL /* start_time is optional */);
232
+ state->slot_two_profile = ddog_Profile_new(sample_types, NULL /* period is optional */, NULL /* start_time is optional */);
233
+
234
+ return TypedData_Wrap_Struct(klass, &stack_recorder_typed_data, state);
235
+ }
236
+
237
+ static void initialize_slot_concurrency_control(struct stack_recorder_state *state) {
220
238
  state->slot_one_mutex = (pthread_mutex_t) PTHREAD_MUTEX_INITIALIZER;
221
239
  state->slot_two_mutex = (pthread_mutex_t) PTHREAD_MUTEX_INITIALIZER;
222
240
 
@@ -225,13 +243,6 @@ static VALUE _native_new(VALUE klass) {
225
243
  if (error) rb_syserr_fail(error, "Unexpected failure during pthread_mutex_lock");
226
244
 
227
245
  state->active_slot = 1;
228
-
229
- // Note: Don't raise exceptions after this point, since it'll lead to libdatadog memory leaking!
230
-
231
- state->slot_one_profile = ddog_Profile_new(sample_types, NULL /* period is optional */, NULL /* start_time is optional */);
232
- state->slot_two_profile = ddog_Profile_new(sample_types, NULL /* period is optional */, NULL /* start_time is optional */);
233
-
234
- return TypedData_Wrap_Struct(klass, &stack_recorder_typed_data, state);
235
246
  }
236
247
 
237
248
  static void stack_recorder_typed_data_free(void *state_ptr) {
@@ -250,9 +261,13 @@ static VALUE _native_serialize(DDTRACE_UNUSED VALUE _self, VALUE recorder_instan
250
261
  struct stack_recorder_state *state;
251
262
  TypedData_Get_Struct(recorder_instance, struct stack_recorder_state, &stack_recorder_typed_data, state);
252
263
 
264
+ ddog_Timespec finish_timestamp = time_now();
265
+ // Need to do this while still holding on to the Global VM Lock; see comments on method for why
266
+ serializer_set_start_timestamp_for_next_profile(state, finish_timestamp);
267
+
253
268
  // We'll release the Global VM Lock while we're calling serialize, so that the Ruby VM can continue to work while this
254
269
  // is pending
255
- struct call_serialize_without_gvl_arguments args = {.state = state, .serialize_ran = false};
270
+ struct call_serialize_without_gvl_arguments args = {.state = state, .finish_timestamp = finish_timestamp, .serialize_ran = false};
256
271
 
257
272
  while (!args.serialize_ran) {
258
273
  // Give the Ruby VM an opportunity to process any pending interruptions (including raising exceptions).
@@ -315,13 +330,22 @@ void record_sample(VALUE recorder_instance, ddog_Sample sample) {
315
330
  sampler_unlock_active_profile(active_slot);
316
331
  }
317
332
 
333
+ void record_endpoint(VALUE recorder_instance, ddog_CharSlice local_root_span_id, ddog_CharSlice endpoint) {
334
+ struct stack_recorder_state *state;
335
+ TypedData_Get_Struct(recorder_instance, struct stack_recorder_state, &stack_recorder_typed_data, state);
336
+
337
+ struct active_slot_pair active_slot = sampler_lock_active_profile(state);
338
+
339
+ ddog_Profile_set_endpoint(active_slot.profile, local_root_span_id, endpoint);
340
+
341
+ sampler_unlock_active_profile(active_slot);
342
+ }
343
+
318
344
  static void *call_serialize_without_gvl(void *call_args) {
319
345
  struct call_serialize_without_gvl_arguments *args = (struct call_serialize_without_gvl_arguments *) call_args;
320
346
 
321
- ddog_Timespec finish_timestamp = time_now();
322
-
323
- args->profile = serializer_flip_active_and_inactive_slots(args->state, finish_timestamp);
324
- args->result = ddog_Profile_serialize(args->profile, &finish_timestamp, NULL /* duration_nanos is optional */);
347
+ args->profile = serializer_flip_active_and_inactive_slots(args->state);
348
+ args->result = ddog_Profile_serialize(args->profile, &args->finish_timestamp, NULL /* duration_nanos is optional */);
325
349
  args->serialize_ran = true;
326
350
 
327
351
  return NULL; // Unused
@@ -360,7 +384,7 @@ static void sampler_unlock_active_profile(struct active_slot_pair active_slot) {
360
384
  if (error != 0) rb_syserr_fail(error, "Unexpected failure in sampler_unlock_active_profile");
361
385
  }
362
386
 
363
- static ddog_Profile *serializer_flip_active_and_inactive_slots(struct stack_recorder_state *state, ddog_Timespec start_timestamp_for_next_profile) {
387
+ static ddog_Profile *serializer_flip_active_and_inactive_slots(struct stack_recorder_state *state) {
364
388
  int error;
365
389
  int previously_active_slot = state->active_slot;
366
390
 
@@ -371,10 +395,6 @@ static ddog_Profile *serializer_flip_active_and_inactive_slots(struct stack_reco
371
395
  pthread_mutex_t *previously_active = (previously_active_slot == 1) ? &state->slot_one_mutex : &state->slot_two_mutex;
372
396
  pthread_mutex_t *previously_inactive = (previously_active_slot == 1) ? &state->slot_two_mutex : &state->slot_one_mutex;
373
397
 
374
- // Before making this profile active, we reset it so that it uses the correct timestamp for its start
375
- ddog_Profile *previously_inactive_profile = (previously_active_slot == 1) ? state->slot_two_profile : state->slot_one_profile;
376
- if (!ddog_Profile_reset(previously_inactive_profile, &start_timestamp_for_next_profile)) rb_raise(rb_eRuntimeError, "Failed to reset profile");
377
-
378
398
  // Release the lock, thus making this slot active
379
399
  error = pthread_mutex_unlock(previously_inactive);
380
400
  if (error) rb_syserr_fail(error, "Unexpected failure during serializer_flip_active_and_inactive_slots for previously_inactive");
@@ -436,3 +456,35 @@ static ddog_Timespec time_now() {
436
456
 
437
457
  return (ddog_Timespec) {.seconds = current_time.tv_sec, .nanoseconds = (uint32_t) current_time.tv_nsec};
438
458
  }
459
+
460
+ // After the Ruby VM forks, this method gets called in the child process to clean up any leftover state from the parent.
461
+ //
462
+ // Assumption: This method gets called BEFORE restarting profiling -- e.g. there are no components attempting to
463
+ // trigger samples at the same time.
464
+ static VALUE _native_reset_after_fork(DDTRACE_UNUSED VALUE self, VALUE recorder_instance) {
465
+ struct stack_recorder_state *state;
466
+ TypedData_Get_Struct(recorder_instance, struct stack_recorder_state, &stack_recorder_typed_data, state);
467
+
468
+ // In case the fork happened halfway through `serializer_flip_active_and_inactive_slots` execution and the
469
+ // resulting state is inconsistent, we make sure to reset it back to the initial state.
470
+ initialize_slot_concurrency_control(state);
471
+
472
+ ddog_Profile_reset(state->slot_one_profile, /* start_time: */ NULL);
473
+ ddog_Profile_reset(state->slot_two_profile, /* start_time: */ NULL);
474
+
475
+ return Qtrue;
476
+ }
477
+
478
+ // Assumption 1: This method is called with the GVL being held, because `ddog_Profile_reset` mutates the profile and should
479
+ // not be interrupted part-way through by a VM fork.
480
+ static void serializer_set_start_timestamp_for_next_profile(struct stack_recorder_state *state, ddog_Timespec timestamp) {
481
+ // Before making this profile active, we reset it so that it uses the correct timestamp for its start
482
+ ddog_Profile *next_profile = (state->active_slot == 1) ? state->slot_two_profile : state->slot_one_profile;
483
+
484
+ if (!ddog_Profile_reset(next_profile, &timestamp)) rb_raise(rb_eRuntimeError, "Failed to reset profile");
485
+ }
486
+
487
+ static VALUE _native_record_endpoint(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance, VALUE local_root_span_id, VALUE endpoint) {
488
+ record_endpoint(recorder_instance, char_slice_from_ruby_string(local_root_span_id), char_slice_from_ruby_string(endpoint));
489
+ return Qtrue;
490
+ }
@@ -35,4 +35,5 @@ static const ddog_ValueType enabled_value_types[] = {
35
35
  #define ENABLED_VALUE_TYPES_COUNT (sizeof(enabled_value_types) / sizeof(ddog_ValueType))
36
36
 
37
37
  void record_sample(VALUE recorder_instance, ddog_Sample sample);
38
+ void record_endpoint(VALUE recorder_instance, ddog_CharSlice local_root_span_id, ddog_CharSlice endpoint);
38
39
  VALUE enforce_recorder_instance(VALUE object);