datadog 2.1.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +101 -1
  3. data/ext/datadog_profiling_loader/extconf.rb +15 -15
  4. data/ext/datadog_profiling_native_extension/clock_id.h +1 -0
  5. data/ext/datadog_profiling_native_extension/clock_id_from_pthread.c +1 -2
  6. data/ext/datadog_profiling_native_extension/clock_id_noop.c +1 -2
  7. data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +132 -44
  8. data/ext/datadog_profiling_native_extension/collectors_discrete_dynamic_sampler.c +49 -26
  9. data/ext/datadog_profiling_native_extension/collectors_discrete_dynamic_sampler.h +34 -4
  10. data/ext/datadog_profiling_native_extension/collectors_idle_sampling_helper.c +4 -0
  11. data/ext/datadog_profiling_native_extension/collectors_stack.c +90 -37
  12. data/ext/datadog_profiling_native_extension/collectors_stack.h +2 -2
  13. data/ext/datadog_profiling_native_extension/collectors_thread_context.c +81 -19
  14. data/ext/datadog_profiling_native_extension/collectors_thread_context.h +1 -0
  15. data/ext/datadog_profiling_native_extension/datadog_ruby_common.c +110 -0
  16. data/ext/datadog_profiling_native_extension/datadog_ruby_common.h +57 -0
  17. data/ext/datadog_profiling_native_extension/extconf.rb +69 -62
  18. data/ext/datadog_profiling_native_extension/heap_recorder.c +34 -6
  19. data/ext/datadog_profiling_native_extension/heap_recorder.h +3 -1
  20. data/ext/datadog_profiling_native_extension/helpers.h +6 -17
  21. data/ext/datadog_profiling_native_extension/http_transport.c +3 -3
  22. data/ext/datadog_profiling_native_extension/libdatadog_helpers.c +0 -86
  23. data/ext/datadog_profiling_native_extension/libdatadog_helpers.h +2 -23
  24. data/ext/datadog_profiling_native_extension/native_extension_helpers.rb +61 -126
  25. data/ext/datadog_profiling_native_extension/private_vm_api_access.c +64 -138
  26. data/ext/datadog_profiling_native_extension/private_vm_api_access.h +17 -11
  27. data/ext/datadog_profiling_native_extension/profiling.c +0 -2
  28. data/ext/datadog_profiling_native_extension/ruby_helpers.c +0 -33
  29. data/ext/datadog_profiling_native_extension/ruby_helpers.h +1 -26
  30. data/ext/datadog_profiling_native_extension/setup_signal_handler.c +1 -1
  31. data/ext/datadog_profiling_native_extension/setup_signal_handler.h +1 -0
  32. data/ext/datadog_profiling_native_extension/stack_recorder.c +27 -8
  33. data/ext/datadog_profiling_native_extension/stack_recorder.h +2 -0
  34. data/ext/datadog_profiling_native_extension/time_helpers.c +0 -15
  35. data/ext/datadog_profiling_native_extension/time_helpers.h +36 -6
  36. data/ext/{datadog_profiling_native_extension → libdatadog_api}/crashtracker.c +20 -7
  37. data/ext/libdatadog_api/datadog_ruby_common.c +110 -0
  38. data/ext/libdatadog_api/datadog_ruby_common.h +57 -0
  39. data/ext/libdatadog_api/extconf.rb +108 -0
  40. data/ext/libdatadog_api/macos_development.md +26 -0
  41. data/ext/libdatadog_extconf_helpers.rb +130 -0
  42. data/lib/datadog/appsec/contrib/graphql/appsec_trace.rb +49 -0
  43. data/lib/datadog/appsec/contrib/graphql/gateway/multiplex.rb +73 -0
  44. data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb +68 -0
  45. data/lib/datadog/appsec/contrib/graphql/integration.rb +41 -0
  46. data/lib/datadog/appsec/contrib/graphql/patcher.rb +37 -0
  47. data/lib/datadog/appsec/contrib/graphql/reactive/multiplex.rb +59 -0
  48. data/lib/datadog/appsec/contrib/rack/gateway/request.rb +1 -1
  49. data/lib/datadog/appsec/contrib/sinatra/patcher.rb +1 -1
  50. data/lib/datadog/appsec/extensions.rb +1 -0
  51. data/lib/datadog/appsec/processor/actions.rb +1 -1
  52. data/lib/datadog/appsec/response.rb +15 -1
  53. data/lib/datadog/appsec.rb +1 -0
  54. data/lib/datadog/core/configuration/components.rb +17 -12
  55. data/lib/datadog/core/configuration/settings.rb +93 -7
  56. data/lib/datadog/core/configuration.rb +3 -17
  57. data/lib/datadog/core/crashtracking/agent_base_url.rb +21 -0
  58. data/lib/datadog/core/crashtracking/component.rb +111 -0
  59. data/lib/datadog/core/crashtracking/tag_builder.rb +39 -0
  60. data/lib/datadog/core/deprecations.rb +58 -0
  61. data/lib/datadog/core/diagnostics/environment_logger.rb +8 -11
  62. data/lib/datadog/core/environment/yjit.rb +5 -0
  63. data/lib/datadog/core/runtime/ext.rb +1 -0
  64. data/lib/datadog/core/runtime/metrics.rb +6 -0
  65. data/lib/datadog/core/telemetry/component.rb +154 -0
  66. data/lib/datadog/core/telemetry/emitter.rb +9 -11
  67. data/lib/datadog/core/telemetry/event.rb +132 -26
  68. data/lib/datadog/core/telemetry/ext.rb +3 -0
  69. data/lib/datadog/core/telemetry/http/adapters/net.rb +11 -13
  70. data/lib/datadog/core/telemetry/http/ext.rb +3 -0
  71. data/lib/datadog/core/telemetry/http/transport.rb +38 -9
  72. data/lib/datadog/core/telemetry/logging.rb +35 -0
  73. data/lib/datadog/core/telemetry/metric.rb +167 -0
  74. data/lib/datadog/core/telemetry/metrics_collection.rb +81 -0
  75. data/lib/datadog/core/telemetry/metrics_manager.rb +81 -0
  76. data/lib/datadog/core/telemetry/request.rb +1 -1
  77. data/lib/datadog/core/telemetry/worker.rb +173 -0
  78. data/lib/datadog/core/utils/at_fork_monkey_patch.rb +102 -0
  79. data/lib/datadog/core/utils/only_once_successful.rb +76 -0
  80. data/lib/datadog/core.rb +2 -19
  81. data/lib/datadog/kit/appsec/events.rb +2 -4
  82. data/lib/datadog/opentelemetry/sdk/propagator.rb +5 -10
  83. data/lib/datadog/opentelemetry/sdk/span_processor.rb +15 -2
  84. data/lib/datadog/opentelemetry/sdk/trace/span.rb +23 -0
  85. data/lib/datadog/profiling/collectors/code_provenance.rb +24 -11
  86. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +17 -17
  87. data/lib/datadog/profiling/collectors/idle_sampling_helper.rb +11 -13
  88. data/lib/datadog/profiling/collectors/info.rb +3 -3
  89. data/lib/datadog/profiling/collectors/thread_context.rb +4 -2
  90. data/lib/datadog/profiling/component.rb +85 -90
  91. data/lib/datadog/profiling/exporter.rb +3 -3
  92. data/lib/datadog/profiling/ext/dir_monkey_patches.rb +410 -0
  93. data/lib/datadog/profiling/ext.rb +21 -21
  94. data/lib/datadog/profiling/flush.rb +1 -1
  95. data/lib/datadog/profiling/http_transport.rb +8 -6
  96. data/lib/datadog/profiling/load_native_extension.rb +5 -5
  97. data/lib/datadog/profiling/preload.rb +1 -1
  98. data/lib/datadog/profiling/profiler.rb +5 -8
  99. data/lib/datadog/profiling/scheduler.rb +31 -25
  100. data/lib/datadog/profiling/tag_builder.rb +2 -2
  101. data/lib/datadog/profiling/tasks/exec.rb +5 -5
  102. data/lib/datadog/profiling/tasks/setup.rb +16 -35
  103. data/lib/datadog/profiling.rb +5 -5
  104. data/lib/datadog/tracing/contrib/action_cable/event.rb +1 -1
  105. data/lib/datadog/tracing/contrib/action_cable/events/broadcast.rb +1 -1
  106. data/lib/datadog/tracing/contrib/action_cable/events/perform_action.rb +1 -1
  107. data/lib/datadog/tracing/contrib/action_cable/events/transmit.rb +1 -1
  108. data/lib/datadog/tracing/contrib/action_mailer/event.rb +4 -6
  109. data/lib/datadog/tracing/contrib/action_mailer/events/deliver.rb +9 -4
  110. data/lib/datadog/tracing/contrib/action_mailer/events/process.rb +3 -2
  111. data/lib/datadog/tracing/contrib/action_view/events/render_partial.rb +1 -5
  112. data/lib/datadog/tracing/contrib/action_view/events/render_template.rb +1 -1
  113. data/lib/datadog/tracing/contrib/active_job/events/discard.rb +1 -1
  114. data/lib/datadog/tracing/contrib/active_job/events/enqueue.rb +1 -1
  115. data/lib/datadog/tracing/contrib/active_job/events/enqueue_at.rb +1 -1
  116. data/lib/datadog/tracing/contrib/active_job/events/enqueue_retry.rb +1 -1
  117. data/lib/datadog/tracing/contrib/active_job/events/perform.rb +1 -1
  118. data/lib/datadog/tracing/contrib/active_job/events/retry_stopped.rb +1 -1
  119. data/lib/datadog/tracing/contrib/active_model_serializers/events/render.rb +1 -1
  120. data/lib/datadog/tracing/contrib/active_model_serializers/events/serialize.rb +1 -1
  121. data/lib/datadog/tracing/contrib/active_record/events/instantiation.rb +1 -1
  122. data/lib/datadog/tracing/contrib/active_record/events/sql.rb +2 -1
  123. data/lib/datadog/tracing/contrib/active_support/cache/event.rb +32 -0
  124. data/lib/datadog/tracing/contrib/active_support/cache/events/cache.rb +156 -0
  125. data/lib/datadog/tracing/contrib/active_support/cache/events.rb +34 -0
  126. data/lib/datadog/tracing/contrib/active_support/cache/instrumentation.rb +45 -41
  127. data/lib/datadog/tracing/contrib/active_support/cache/patcher.rb +17 -40
  128. data/lib/datadog/tracing/contrib/active_support/cache/redis.rb +4 -1
  129. data/lib/datadog/tracing/contrib/active_support/notifications/event.rb +29 -6
  130. data/lib/datadog/tracing/contrib/active_support/notifications/subscriber.rb +16 -4
  131. data/lib/datadog/tracing/contrib/active_support/notifications/subscription.rb +33 -29
  132. data/lib/datadog/tracing/contrib/analytics.rb +5 -0
  133. data/lib/datadog/tracing/contrib/ext.rb +14 -0
  134. data/lib/datadog/tracing/contrib/graphql/configuration/settings.rb +5 -0
  135. data/lib/datadog/tracing/contrib/graphql/patcher.rb +8 -2
  136. data/lib/datadog/tracing/contrib/graphql/unified_trace.rb +166 -0
  137. data/lib/datadog/tracing/contrib/graphql/unified_trace_patcher.rb +28 -0
  138. data/lib/datadog/tracing/contrib/kafka/consumer_event.rb +1 -1
  139. data/lib/datadog/tracing/contrib/kafka/consumer_group_event.rb +1 -1
  140. data/lib/datadog/tracing/contrib/kafka/event.rb +1 -1
  141. data/lib/datadog/tracing/contrib/kafka/events/connection/request.rb +3 -3
  142. data/lib/datadog/tracing/contrib/kafka/events/consumer/process_batch.rb +3 -3
  143. data/lib/datadog/tracing/contrib/kafka/events/consumer/process_message.rb +3 -3
  144. data/lib/datadog/tracing/contrib/kafka/events/consumer_group/heartbeat.rb +3 -3
  145. data/lib/datadog/tracing/contrib/kafka/events/produce_operation/send_messages.rb +3 -3
  146. data/lib/datadog/tracing/contrib/kafka/events/producer/deliver_messages.rb +3 -3
  147. data/lib/datadog/tracing/contrib/lograge/patcher.rb +16 -0
  148. data/lib/datadog/tracing/contrib/mysql2/configuration/settings.rb +5 -0
  149. data/lib/datadog/tracing/contrib/mysql2/instrumentation.rb +17 -13
  150. data/lib/datadog/tracing/contrib/pg/configuration/settings.rb +5 -0
  151. data/lib/datadog/tracing/contrib/pg/instrumentation.rb +4 -1
  152. data/lib/datadog/tracing/contrib/propagation/sql_comment/ext.rb +28 -0
  153. data/lib/datadog/tracing/contrib/propagation/sql_comment/mode.rb +5 -1
  154. data/lib/datadog/tracing/contrib/propagation/sql_comment.rb +22 -10
  155. data/lib/datadog/tracing/contrib/racecar/event.rb +2 -2
  156. data/lib/datadog/tracing/contrib/rails/ext.rb +9 -0
  157. data/lib/datadog/tracing/contrib/rails/patcher.rb +7 -0
  158. data/lib/datadog/tracing/contrib/rails/runner.rb +95 -0
  159. data/lib/datadog/tracing/contrib/trilogy/configuration/settings.rb +5 -0
  160. data/lib/datadog/tracing/contrib/trilogy/instrumentation.rb +4 -1
  161. data/lib/datadog/tracing/diagnostics/environment_logger.rb +14 -16
  162. data/lib/datadog/tracing/distributed/b3_multi.rb +1 -1
  163. data/lib/datadog/tracing/distributed/b3_single.rb +3 -1
  164. data/lib/datadog/tracing/distributed/datadog.rb +2 -2
  165. data/lib/datadog/tracing/distributed/propagation.rb +9 -2
  166. data/lib/datadog/tracing/distributed/trace_context.rb +3 -2
  167. data/lib/datadog/tracing/metadata/errors.rb +9 -1
  168. data/lib/datadog/tracing/metadata/ext.rb +4 -0
  169. data/lib/datadog/tracing/pipeline/span_filter.rb +2 -2
  170. data/lib/datadog/tracing/span.rb +9 -2
  171. data/lib/datadog/tracing/span_event.rb +41 -0
  172. data/lib/datadog/tracing/span_operation.rb +9 -4
  173. data/lib/datadog/tracing/trace_operation.rb +7 -3
  174. data/lib/datadog/tracing/trace_segment.rb +4 -1
  175. data/lib/datadog/tracing/tracer.rb +9 -2
  176. data/lib/datadog/tracing/transport/serializable_trace.rb +3 -0
  177. data/lib/datadog/tracing.rb +5 -1
  178. data/lib/datadog/version.rb +2 -2
  179. metadata +43 -12
  180. data/lib/datadog/core/telemetry/client.rb +0 -95
  181. data/lib/datadog/core/telemetry/heartbeat.rb +0 -33
  182. data/lib/datadog/profiling/crashtracker.rb +0 -91
  183. data/lib/datadog/profiling/ext/forking.rb +0 -98
@@ -20,6 +20,9 @@
20
20
 
21
21
  #define EMA_SMOOTHING_FACTOR 0.6
22
22
 
23
+ static void maybe_readjust(discrete_dynamic_sampler *sampler, long now_ns);
24
+ static inline bool should_readjust(discrete_dynamic_sampler *sampler, coarse_instant now);
25
+
23
26
  void discrete_dynamic_sampler_init(discrete_dynamic_sampler *sampler, const char *debug_name, long now_ns) {
24
27
  sampler->debug_name = debug_name;
25
28
  discrete_dynamic_sampler_set_overhead_target_percentage(sampler, BASE_OVERHEAD_PCT, now_ns);
@@ -54,27 +57,25 @@ void discrete_dynamic_sampler_set_overhead_target_percentage(discrete_dynamic_sa
54
57
  return discrete_dynamic_sampler_reset(sampler, now_ns);
55
58
  }
56
59
 
57
- static void maybe_readjust(discrete_dynamic_sampler *sampler, long now);
58
-
59
- bool discrete_dynamic_sampler_should_sample(discrete_dynamic_sampler *sampler, long now_ns) {
60
+ // NOTE: See header for an explanation of when this should get used
61
+ __attribute__((warn_unused_result))
62
+ bool discrete_dynamic_sampler_should_sample(discrete_dynamic_sampler *sampler) {
60
63
  // For efficiency reasons we don't do true random sampling but rather systematic
61
64
  // sampling following a sample interval/skip. This can be biased and hide patterns
62
- // but the dynamic interval and rather indeterministic pattern of allocations in
65
+ // but the dynamic interval and rather nondeterministic pattern of allocations in
63
66
  // most real applications should help reduce the bias impact.
64
67
  sampler->events_since_last_sample++;
65
68
  sampler->events_since_last_readjustment++;
66
- bool should_sample = sampler->sampling_interval > 0 && sampler->events_since_last_sample >= sampler->sampling_interval;
67
69
 
68
- if (should_sample) {
69
- sampler->sample_start_time_ns = now_ns;
70
- } else {
71
- // check if we should readjust our sampler after this event, even if we didn't sample it
72
- maybe_readjust(sampler, now_ns);
73
- }
70
+ return sampler->sampling_interval > 0 && sampler->events_since_last_sample >= sampler->sampling_interval;
71
+ }
74
72
 
75
- return should_sample;
73
+ // NOTE: See header for an explanation of when this should get used
74
+ void discrete_dynamic_sampler_before_sample(discrete_dynamic_sampler *sampler, long now_ns) {
75
+ sampler->sample_start_time_ns = now_ns;
76
76
  }
77
77
 
78
+ // NOTE: See header for an explanation of when this should get used
78
79
  long discrete_dynamic_sampler_after_sample(discrete_dynamic_sampler *sampler, long now_ns) {
79
80
  long last_sampling_time_ns = sampler->sample_start_time_ns == 0 ? 0 : long_max_of(0, now_ns - sampler->sample_start_time_ns);
80
81
  sampler->samples_since_last_readjustment++;
@@ -95,6 +96,11 @@ size_t discrete_dynamic_sampler_events_since_last_sample(discrete_dynamic_sample
95
96
  return sampler->events_since_last_sample;
96
97
  }
97
98
 
99
+ // NOTE: See header for an explanation of when this should get used
100
+ bool discrete_dynamic_sampler_skipped_sample(discrete_dynamic_sampler *sampler, coarse_instant now) {
101
+ return should_readjust(sampler, now);
102
+ }
103
+
98
104
  static double ewma_adj_window(double latest_value, double avg, long current_window_time_ns, bool is_first) {
99
105
  if (is_first) {
100
106
  return latest_value;
@@ -109,19 +115,27 @@ static double ewma_adj_window(double latest_value, double avg, long current_wind
109
115
  return (1-alpha) * avg + alpha * latest_value;
110
116
  }
111
117
 
112
- static void maybe_readjust(discrete_dynamic_sampler *sampler, long now) {
113
- long this_window_time_ns = sampler->last_readjust_time_ns == 0 ? ADJUSTMENT_WINDOW_NS : now - sampler->last_readjust_time_ns;
118
+ static void maybe_readjust(discrete_dynamic_sampler *sampler, long now_ns) {
119
+ if (should_readjust(sampler, to_coarse_instant(now_ns))) discrete_dynamic_sampler_readjust(sampler, now_ns);
120
+ }
121
+
122
+ static inline bool should_readjust(discrete_dynamic_sampler *sampler, coarse_instant now) {
123
+ long this_window_time_ns =
124
+ sampler->last_readjust_time_ns == 0 ? ADJUSTMENT_WINDOW_NS : now.timestamp_ns - sampler->last_readjust_time_ns;
114
125
 
115
126
  bool should_readjust_based_on_time = this_window_time_ns >= ADJUSTMENT_WINDOW_NS;
116
127
  bool should_readjust_based_on_samples = sampler->samples_since_last_readjustment >= ADJUSTMENT_WINDOW_SAMPLES;
117
128
 
118
- if (!should_readjust_based_on_time && !should_readjust_based_on_samples) {
119
- // not enough time or samples have passed to perform a readjustment
120
- return;
121
- }
129
+ return should_readjust_based_on_time || should_readjust_based_on_samples;
130
+ }
131
+
132
+ // NOTE: This method ASSUMES should_readjust == true
133
+ // NOTE: See header for an explanation of when this should get used
134
+ void discrete_dynamic_sampler_readjust(discrete_dynamic_sampler *sampler, long now_ns) {
135
+ long this_window_time_ns = sampler->last_readjust_time_ns == 0 ? ADJUSTMENT_WINDOW_NS : now_ns - sampler->last_readjust_time_ns;
122
136
 
123
- if (this_window_time_ns == 0) {
124
- // should not be possible given previous condition but lets protect against div by 0 below.
137
+ if (this_window_time_ns <= 0) {
138
+ // should not be possible given should_readjust but lets protect against div by 0 below.
125
139
  return;
126
140
  }
127
141
 
@@ -143,7 +157,7 @@ static void maybe_readjust(discrete_dynamic_sampler *sampler, long now) {
143
157
  // Lets update our average sampling time per event
144
158
  long avg_sampling_time_in_window_ns = sampler->samples_since_last_readjustment == 0 ? 0 : sampler->sampling_time_since_last_readjustment_ns / sampler->samples_since_last_readjustment;
145
159
  if (avg_sampling_time_in_window_ns > sampler->max_sampling_time_ns) {
146
- // If the average sampling time in the previous window was deemed unnacceptable, clamp it to the
160
+ // If the average sampling time in the previous window was deemed unacceptable, clamp it to the
147
161
  // maximum acceptable value and register this operation in our counter.
148
162
  // NOTE: This is important so that events like suspensions or system overloads do not lead us to
149
163
  // learn arbitrarily big sampling times which may then result in us not sampling anything
@@ -255,9 +269,7 @@ static void maybe_readjust(discrete_dynamic_sampler *sampler, long now) {
255
269
  double num_this_windows_in_60s = 60 * 1e9 / this_window_time_ns;
256
270
  double real_total_sampling_time_in_60s = sampler->sampling_time_since_last_readjustment_ns * num_this_windows_in_60s / 1e9;
257
271
 
258
- const char* readjustment_reason = should_readjust_based_on_time ? "time" : "samples";
259
-
260
- fprintf(stderr, "[dds.%s] readjusting due to %s...\n", sampler->debug_name, readjustment_reason);
272
+ fprintf(stderr, "[dds.%s] readjusting...\n", sampler->debug_name);
261
273
  fprintf(stderr, "events_since_last_readjustment=%ld\n", sampler->events_since_last_readjustment);
262
274
  fprintf(stderr, "samples_since_last_readjustment=%ld\n", sampler->samples_since_last_readjustment);
263
275
  fprintf(stderr, "this_window_time=%ld\n", this_window_time_ns);
@@ -286,7 +298,7 @@ static void maybe_readjust(discrete_dynamic_sampler *sampler, long now) {
286
298
  sampler->events_since_last_readjustment = 0;
287
299
  sampler->samples_since_last_readjustment = 0;
288
300
  sampler->sampling_time_since_last_readjustment_ns = 0;
289
- sampler->last_readjust_time_ns = now;
301
+ sampler->last_readjust_time_ns = now_ns;
290
302
  sampler->has_completed_full_adjustment_window = true;
291
303
  }
292
304
 
@@ -359,6 +371,10 @@ static VALUE _native_new(VALUE klass) {
359
371
  }
360
372
  discrete_dynamic_sampler_init(&state->sampler, "test sampler", now_ns);
361
373
 
374
+ // Note: As of this writing, no new Ruby objects get created and stored in the state. If that ever changes, remember
375
+ // to keep them on the stack and mark them with RB_GC_GUARD -- otherwise it's possible for a GC to run and
376
+ // since the instance representing the state does not yet exist, such objects will not get marked.
377
+
362
378
  return TypedData_Wrap_Struct(klass, &sampler_typed_data, state);
363
379
  }
364
380
 
@@ -402,7 +418,14 @@ VALUE _native_should_sample(VALUE self, VALUE now_ns) {
402
418
  sampler_state *state;
403
419
  TypedData_Get_Struct(self, sampler_state, &sampler_typed_data, state);
404
420
 
405
- return discrete_dynamic_sampler_should_sample(&state->sampler, NUM2LONG(now_ns)) ? Qtrue : Qfalse;
421
+ if (discrete_dynamic_sampler_should_sample(&state->sampler)) {
422
+ discrete_dynamic_sampler_before_sample(&state->sampler, NUM2LONG(now_ns));
423
+ return Qtrue;
424
+ } else {
425
+ bool needs_readjust = discrete_dynamic_sampler_skipped_sample(&state->sampler, to_coarse_instant(NUM2LONG(now_ns)));
426
+ if (needs_readjust) discrete_dynamic_sampler_readjust(&state->sampler, NUM2LONG(now_ns));
427
+ return Qfalse;
428
+ }
406
429
  }
407
430
 
408
431
  VALUE _native_after_sample(VALUE self, VALUE now_ns) {
@@ -5,6 +5,8 @@
5
5
 
6
6
  #include <ruby.h>
7
7
 
8
+ #include "time_helpers.h"
9
+
8
10
  // A sampler that will sample discrete events based on the overhead of their
9
11
  // sampling.
10
12
  //
@@ -62,7 +64,6 @@ typedef struct discrete_dynamic_sampler {
62
64
  unsigned long sampling_time_clamps;
63
65
  } discrete_dynamic_sampler;
64
66
 
65
-
66
67
  // Init a new sampler with sane defaults.
67
68
  void discrete_dynamic_sampler_init(discrete_dynamic_sampler *sampler, const char *debug_name, long now_ns);
68
69
 
@@ -80,9 +81,38 @@ void discrete_dynamic_sampler_set_overhead_target_percentage(discrete_dynamic_sa
80
81
  // @return True if the event associated with this decision should be sampled, false
81
82
  // otherwise.
82
83
  //
83
- // NOTE: If true is returned we implicitly assume the start of a sampling operation
84
- // and it is expected that a follow-up after_sample call is issued.
85
- bool discrete_dynamic_sampler_should_sample(discrete_dynamic_sampler *sampler, long now_ns);
84
+ // IMPORTANT: A call to this method MUST be followed by a call to either
85
+ // `discrete_dynamic_sampler_before_sample` if return is `true` or
86
+ // `discrete_dynamic_sampler_skipped_sample` if return is `false`.
87
+ //
88
+ // In the past, this method took care of before_sample/skipped_sample/readjust as well.
89
+ // We've had to split it up because we wanted to both use coarse time and regular monotonic time depending on the
90
+ // situation, but also wanted time to come as an argument from the outside.
91
+ //
92
+ // TL;DR here's how they should be used as Ruby code:
93
+ // if discrete_dynamic_sampler_should_sample
94
+ // discrete_dynamic_sampler_before_sample(monotonic_wall_time_now_ns())
95
+ // else
96
+ // needs_readjust = discrete_dynamic_sampler_skipped_sample(monotonic_coarse_wall_time_now_ns())
97
+ // discrete_dynamic_sampler_readjust(monotonic_wall_time_now_ns()) if needs_readjust
98
+ // end
99
+ __attribute__((warn_unused_result))
100
+ bool discrete_dynamic_sampler_should_sample(discrete_dynamic_sampler *sampler);
101
+
102
+ // Signal the start of a sampling operation.
103
+ // MUST be called after `discrete_dynamic_sampler_should_sample` returns `true`.
104
+ void discrete_dynamic_sampler_before_sample(discrete_dynamic_sampler *sampler, long now_ns);
105
+
106
+ // Signals that sampling did not happen
107
+ // MUST be called after `discrete_dynamic_sampler_skipped_sample` returns `false`.
108
+ //
109
+ // @return True if the sampler needs to be readjusted.
110
+ //
111
+ // IMPORTANT: A call to this method MUST be followed by a call to `discrete_dynamic_sampler_readjust` if return is `true`.
112
+ __attribute__((warn_unused_result))
113
+ bool discrete_dynamic_sampler_skipped_sample(discrete_dynamic_sampler *sampler, coarse_instant now);
114
+
115
+ void discrete_dynamic_sampler_readjust(discrete_dynamic_sampler *sampler, long now_ns);
86
116
 
87
117
  // Signal the end of a sampling operation.
88
118
  //
@@ -83,6 +83,10 @@ static VALUE _native_new(VALUE klass) {
83
83
 
84
84
  reset_state(state);
85
85
 
86
+ // Note: As of this writing, no new Ruby objects get created and stored in the state. If that ever changes, remember
87
+ // to keep them on the stack and mark them with RB_GC_GUARD -- otherwise it's possible for a GC to run and
88
+ // since the instance representing the state does not yet exist, such objects will not get marked.
89
+
86
90
  return TypedData_Wrap_Struct(klass, &idle_sampling_helper_typed_data, state);
87
91
  }
88
92
 
@@ -15,11 +15,9 @@ static VALUE missing_string = Qnil;
15
15
 
16
16
  // Used as scratch space during sampling
17
17
  struct sampling_buffer {
18
- unsigned int max_frames;
19
- VALUE *stack_buffer;
20
- int *lines_buffer;
21
- bool *is_ruby_frame;
18
+ uint16_t max_frames;
22
19
  ddog_prof_Location *locations;
20
+ frame_info *stack_buffer;
23
21
  }; // Note: typedef'd in the header to sampling_buffer
24
22
 
25
23
  static VALUE _native_sample(
@@ -33,7 +31,13 @@ static VALUE _native_sample(
33
31
  VALUE in_gc
34
32
  );
35
33
  static void maybe_add_placeholder_frames_omitted(VALUE thread, sampling_buffer* buffer, char *frames_omitted_message, int frames_omitted_message_size);
36
- static void record_placeholder_stack_in_native_code(sampling_buffer* buffer, VALUE recorder_instance, sample_values values, sample_labels labels);
34
+ static void record_placeholder_stack_in_native_code(VALUE recorder_instance, sample_values values, sample_labels labels);
35
+ static void maybe_trim_template_random_ids(ddog_CharSlice *name_slice, ddog_CharSlice *filename_slice);
36
+
37
+ // These two functions are exposed as symbols by the VM but are not in any header.
38
+ // Their signatures actually take a `const rb_iseq_t *iseq` but it gets casted back and forth between VALUE.
39
+ extern VALUE rb_iseq_path(const VALUE);
40
+ extern VALUE rb_iseq_base_label(const VALUE);
37
41
 
38
42
  void collectors_stack_init(VALUE profiling_module) {
39
43
  VALUE collectors_module = rb_define_module_under(profiling_module, "Collectors");
@@ -64,12 +68,16 @@ static VALUE _native_sample(
64
68
  ENFORCE_TYPE(numeric_labels_array, T_ARRAY);
65
69
 
66
70
  VALUE zero = INT2NUM(0);
71
+ VALUE heap_sample = rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("heap_sample"), Qfalse);
72
+ ENFORCE_BOOLEAN(heap_sample);
67
73
  sample_values values = {
68
74
  .cpu_time_ns = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("cpu-time"), zero)),
69
75
  .cpu_or_wall_samples = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("cpu-samples"), zero)),
70
76
  .wall_time_ns = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("wall-time"), zero)),
71
77
  .alloc_samples = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("alloc-samples"), zero)),
78
+ .alloc_samples_unscaled = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("alloc-samples-unscaled"), zero)),
72
79
  .timeline_wall_time_ns = NUM2UINT(rb_hash_lookup2(metric_values_hash, rb_str_new_cstr("timeline"), zero)),
80
+ .heap_sample = heap_sample == Qtrue,
73
81
  };
74
82
 
75
83
  long labels_count = RARRAY_LEN(labels_array) + RARRAY_LEN(numeric_labels_array);
@@ -100,13 +108,13 @@ static VALUE _native_sample(
100
108
  int max_frames_requested = NUM2INT(max_frames);
101
109
  if (max_frames_requested < 0) rb_raise(rb_eArgError, "Invalid max_frames: value must not be negative");
102
110
 
103
- sampling_buffer *buffer = sampling_buffer_new(max_frames_requested);
111
+ ddog_prof_Location *locations = ruby_xcalloc(max_frames_requested, sizeof(ddog_prof_Location));
112
+ sampling_buffer *buffer = sampling_buffer_new(max_frames_requested, locations);
104
113
 
105
114
  ddog_prof_Slice_Label slice_labels = {.ptr = labels, .len = labels_count};
106
115
 
107
116
  if (in_gc == Qtrue) {
108
117
  record_placeholder_stack(
109
- buffer,
110
118
  recorder_instance,
111
119
  values,
112
120
  (sample_labels) {.labels = slice_labels, .state_label = state_label},
@@ -122,6 +130,7 @@ static VALUE _native_sample(
122
130
  );
123
131
  }
124
132
 
133
+ ruby_xfree(locations);
125
134
  sampling_buffer_free(buffer);
126
135
 
127
136
  return Qtrue;
@@ -149,22 +158,28 @@ void sample_thread(
149
158
  thread,
150
159
  0 /* stack starting depth */,
151
160
  buffer->max_frames,
152
- buffer->stack_buffer,
153
- buffer->lines_buffer,
154
- buffer->is_ruby_frame
161
+ buffer->stack_buffer
155
162
  );
156
163
 
157
164
  if (captured_frames == PLACEHOLDER_STACK_IN_NATIVE_CODE) {
158
- record_placeholder_stack_in_native_code(buffer, recorder_instance, values, labels);
165
+ record_placeholder_stack_in_native_code(recorder_instance, values, labels);
159
166
  return;
160
167
  }
161
168
 
169
+ // if (captured_frames > 0) {
170
+ // int cache_hits = 0;
171
+ // for (int i = 0; i < captured_frames; i++) {
172
+ // if (buffer->stack_buffer[i].same_frame) cache_hits++;
173
+ // }
174
+ // fprintf(stderr, "Sampling cache hits: %f\n", ((double) cache_hits / captured_frames) * 100);
175
+ // }
176
+
162
177
  // Ruby does not give us path and line number for methods implemented using native code.
163
178
  // The convention in Kernel#caller_locations is to instead use the path and line number of the first Ruby frame
164
179
  // on the stack that is below (e.g. directly or indirectly has called) the native method.
165
180
  // Thus, we keep that frame here to able to replicate that behavior.
166
- // (This is why we also iterate the sampling buffers backwards below -- so that it's easier to keep the last_ruby_frame)
167
- VALUE last_ruby_frame = Qnil;
181
+ // (This is why we also iterate the sampling buffers backwards below -- so that it's easier to keep the last_ruby_frame_filename)
182
+ VALUE last_ruby_frame_filename = Qnil;
168
183
  int last_ruby_line = 0;
169
184
 
170
185
  ddog_prof_Label *state_label = labels.state_label;
@@ -180,16 +195,16 @@ void sample_thread(
180
195
  VALUE name, filename;
181
196
  int line;
182
197
 
183
- if (buffer->is_ruby_frame[i]) {
184
- last_ruby_frame = buffer->stack_buffer[i];
185
- last_ruby_line = buffer->lines_buffer[i];
198
+ if (buffer->stack_buffer[i].is_ruby_frame) {
199
+ name = rb_iseq_base_label(buffer->stack_buffer[i].as.ruby_frame.iseq);
200
+ filename = rb_iseq_path(buffer->stack_buffer[i].as.ruby_frame.iseq);
201
+ line = buffer->stack_buffer[i].as.ruby_frame.line;
186
202
 
187
- name = rb_profile_frame_base_label(buffer->stack_buffer[i]);
188
- filename = rb_profile_frame_path(buffer->stack_buffer[i]);
189
- line = buffer->lines_buffer[i];
203
+ last_ruby_frame_filename = filename;
204
+ last_ruby_line = line;
190
205
  } else {
191
- name = ddtrace_rb_profile_frame_method_name(buffer->stack_buffer[i]);
192
- filename = NIL_P(last_ruby_frame) ? Qnil : rb_profile_frame_path(last_ruby_frame);
206
+ name = rb_id2str(buffer->stack_buffer[i].as.native_frame.method_id);
207
+ filename = last_ruby_frame_filename;
193
208
  line = last_ruby_line;
194
209
  }
195
210
 
@@ -199,6 +214,8 @@ void sample_thread(
199
214
  ddog_CharSlice name_slice = char_slice_from_ruby_string(name);
200
215
  ddog_CharSlice filename_slice = char_slice_from_ruby_string(filename);
201
216
 
217
+ maybe_trim_template_random_ids(&name_slice, &filename_slice);
218
+
202
219
  bool top_of_the_stack = i == 0;
203
220
 
204
221
  // When there's only wall-time in a sample, this means that the thread was not active in the sampled period.
@@ -207,7 +224,7 @@ void sample_thread(
207
224
  // approximation, and in the future we hope to replace this with a more accurate approach (such as using the
208
225
  // GVL instrumentation API.)
209
226
  if (top_of_the_stack && only_wall_time) {
210
- if (!buffer->is_ruby_frame[i]) {
227
+ if (!buffer->stack_buffer[i].is_ruby_frame) {
211
228
  // We know that known versions of Ruby implement these using native code; thus if we find a method with the
212
229
  // same name that is not native code, we ignore it, as it's probably a user method that coincidentally
213
230
  // has the same name. Thus, even though "matching just by method name" is kinda weak,
@@ -268,6 +285,43 @@ void sample_thread(
268
285
  );
269
286
  }
270
287
 
288
+ // Rails's ActionView likes to dynamically generate method names with suffixed hashes/ids, resulting in methods with
289
+ // names such as:
290
+ // * "_app_views_layouts_explore_html_haml__2304485752546535910_211320" (__number_number suffix -- two underscores)
291
+ // * "_app_views_articles_index_html_erb___2022809201779434309_12900" (___number_number suffix -- three underscores)
292
+ // This makes these stacks not aggregate well, as well as being not-very-useful data.
293
+ // (Reference:
294
+ // https://github.com/rails/rails/blob/4fa56814f18fd3da49c83931fa773caa727d8096/actionview/lib/action_view/template.rb#L389
295
+ // The two vs three underscores happen when @identifier.hash is negative in that method: the "-" gets replaced with
296
+ // the extra "_".)
297
+ //
298
+ // This method trims these suffixes, so that we keep less data + the names correctly aggregate together.
299
+ static void maybe_trim_template_random_ids(ddog_CharSlice *name_slice, ddog_CharSlice *filename_slice) {
300
+ // Check filename doesn't end with ".rb"; templates are usually along the lines of .html.erb/.html.haml/...
301
+ if (filename_slice->len < 3 || memcmp(filename_slice->ptr + filename_slice->len - 3, ".rb", 3) == 0) return;
302
+
303
+ int pos = name_slice->len - 1;
304
+
305
+ // Let's match on something__number_number:
306
+ // Find start of id suffix from the end...
307
+ if (name_slice->ptr[pos] < '0' || name_slice->ptr[pos] > '9') return;
308
+
309
+ // ...now match a bunch of numbers and interspersed underscores
310
+ for (int underscores = 0; pos >= 0 && underscores < 2; pos--) {
311
+ if (name_slice->ptr[pos] == '_') underscores++;
312
+ else if (name_slice->ptr[pos] < '0' || name_slice->ptr[pos] > '9') return;
313
+ }
314
+
315
+ // Make sure there's something left before the underscores (hence the <= instead of <) + match the last underscore
316
+ if (pos <= 0 || name_slice->ptr[pos] != '_') return;
317
+
318
+ // Does it have the optional third underscore? If so, remove it as well
319
+ if (pos > 1 && name_slice->ptr[pos-1] == '_') pos--;
320
+
321
+ // If we got here, we matched on our pattern. Let's slice the length of the string to exclude it.
322
+ name_slice->len = pos;
323
+ }
324
+
271
325
  static void maybe_add_placeholder_frames_omitted(VALUE thread, sampling_buffer* buffer, char *frames_omitted_message, int frames_omitted_message_size) {
272
326
  ptrdiff_t frames_omitted = stack_depth_for(thread) - buffer->max_frames;
273
327
 
@@ -311,13 +365,11 @@ static void maybe_add_placeholder_frames_omitted(VALUE thread, sampling_buffer*
311
365
  // with one containing a placeholder frame, so that these threads are properly represented in the UX.
312
366
 
313
367
  static void record_placeholder_stack_in_native_code(
314
- sampling_buffer* buffer,
315
368
  VALUE recorder_instance,
316
369
  sample_values values,
317
370
  sample_labels labels
318
371
  ) {
319
372
  record_placeholder_stack(
320
- buffer,
321
373
  recorder_instance,
322
374
  values,
323
375
  labels,
@@ -326,36 +378,39 @@ static void record_placeholder_stack_in_native_code(
326
378
  }
327
379
 
328
380
  void record_placeholder_stack(
329
- sampling_buffer* buffer,
330
381
  VALUE recorder_instance,
331
382
  sample_values values,
332
383
  sample_labels labels,
333
384
  ddog_CharSlice placeholder_stack
334
385
  ) {
335
- ddog_prof_Function placeholder = {.name = DDOG_CHARSLICE_C(""), .filename = placeholder_stack};
336
- buffer->locations[0] = (ddog_prof_Location) {.function = placeholder, .line = 0};
386
+ ddog_prof_Location placeholder_location = {
387
+ .function = {.name = DDOG_CHARSLICE_C(""), .filename = placeholder_stack},
388
+ .line = 0,
389
+ };
337
390
 
338
391
  record_sample(
339
392
  recorder_instance,
340
- (ddog_prof_Slice_Location) {.ptr = buffer->locations, .len = 1},
393
+ (ddog_prof_Slice_Location) {.ptr = &placeholder_location, .len = 1},
341
394
  values,
342
395
  labels
343
396
  );
344
397
  }
345
398
 
346
- sampling_buffer *sampling_buffer_new(unsigned int max_frames) {
399
+ uint16_t sampling_buffer_check_max_frames(int max_frames) {
347
400
  if (max_frames < 5) rb_raise(rb_eArgError, "Invalid max_frames: value must be >= 5");
348
401
  if (max_frames > MAX_FRAMES_LIMIT) rb_raise(rb_eArgError, "Invalid max_frames: value must be <= " MAX_FRAMES_LIMIT_AS_STRING);
402
+ return max_frames;
403
+ }
404
+
405
+ sampling_buffer *sampling_buffer_new(uint16_t max_frames, ddog_prof_Location *locations) {
406
+ sampling_buffer_check_max_frames(max_frames);
349
407
 
350
408
  // Note: never returns NULL; if out of memory, it calls the Ruby out-of-memory handlers
351
409
  sampling_buffer* buffer = ruby_xcalloc(1, sizeof(sampling_buffer));
352
410
 
353
411
  buffer->max_frames = max_frames;
354
-
355
- buffer->stack_buffer = ruby_xcalloc(max_frames, sizeof(VALUE));
356
- buffer->lines_buffer = ruby_xcalloc(max_frames, sizeof(int));
357
- buffer->is_ruby_frame = ruby_xcalloc(max_frames, sizeof(bool));
358
- buffer->locations = ruby_xcalloc(max_frames, sizeof(ddog_prof_Location));
412
+ buffer->locations = locations;
413
+ buffer->stack_buffer = ruby_xcalloc(max_frames, sizeof(frame_info));
359
414
 
360
415
  return buffer;
361
416
  }
@@ -363,10 +418,8 @@ sampling_buffer *sampling_buffer_new(unsigned int max_frames) {
363
418
  void sampling_buffer_free(sampling_buffer *buffer) {
364
419
  if (buffer == NULL) rb_raise(rb_eArgError, "sampling_buffer_free called with NULL buffer");
365
420
 
421
+ // buffer->locations are owned by whoever called sampling_buffer_new, not us
366
422
  ruby_xfree(buffer->stack_buffer);
367
- ruby_xfree(buffer->lines_buffer);
368
- ruby_xfree(buffer->is_ruby_frame);
369
- ruby_xfree(buffer->locations);
370
423
 
371
424
  ruby_xfree(buffer);
372
425
  }
@@ -17,11 +17,11 @@ void sample_thread(
17
17
  sample_labels labels
18
18
  );
19
19
  void record_placeholder_stack(
20
- sampling_buffer* buffer,
21
20
  VALUE recorder_instance,
22
21
  sample_values values,
23
22
  sample_labels labels,
24
23
  ddog_CharSlice placeholder_stack
25
24
  );
26
- sampling_buffer *sampling_buffer_new(unsigned int max_frames);
25
+ uint16_t sampling_buffer_check_max_frames(int max_frames);
26
+ sampling_buffer *sampling_buffer_new(uint16_t max_frames, ddog_prof_Location *locations);
27
27
  void sampling_buffer_free(sampling_buffer *buffer);