ddtrace 1.21.1 → 1.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +39 -1
  3. data/ext/datadog_profiling_native_extension/collectors_cpu_and_wall_time_worker.c +32 -12
  4. data/ext/datadog_profiling_native_extension/collectors_thread_context.c +5 -2
  5. data/ext/datadog_profiling_native_extension/heap_recorder.c +45 -3
  6. data/ext/datadog_profiling_native_extension/heap_recorder.h +7 -1
  7. data/ext/datadog_profiling_native_extension/http_transport.c +5 -5
  8. data/ext/datadog_profiling_native_extension/native_extension_helpers.rb +1 -1
  9. data/ext/datadog_profiling_native_extension/stack_recorder.c +7 -9
  10. data/lib/datadog/appsec/contrib/rack/request_middleware.rb +43 -13
  11. data/lib/datadog/appsec/event.rb +1 -1
  12. data/lib/datadog/core/configuration/components.rb +2 -1
  13. data/lib/datadog/core/configuration/option.rb +7 -5
  14. data/lib/datadog/core/configuration/settings.rb +38 -17
  15. data/lib/datadog/core/configuration.rb +20 -4
  16. data/lib/datadog/core/environment/platform.rb +7 -1
  17. data/lib/datadog/core/remote/client/capabilities.rb +1 -1
  18. data/lib/datadog/core/remote/transport/http/config.rb +1 -1
  19. data/lib/datadog/core/telemetry/client.rb +18 -10
  20. data/lib/datadog/core/telemetry/emitter.rb +9 -13
  21. data/lib/datadog/core/telemetry/event.rb +247 -57
  22. data/lib/datadog/core/telemetry/ext.rb +1 -0
  23. data/lib/datadog/core/telemetry/heartbeat.rb +1 -3
  24. data/lib/datadog/core/telemetry/http/ext.rb +4 -1
  25. data/lib/datadog/core/telemetry/http/transport.rb +9 -4
  26. data/lib/datadog/core/telemetry/request.rb +59 -0
  27. data/lib/datadog/profiling/collectors/code_provenance.rb +10 -4
  28. data/lib/datadog/profiling/collectors/cpu_and_wall_time_worker.rb +25 -0
  29. data/lib/datadog/profiling/component.rb +23 -15
  30. data/lib/datadog/profiling/load_native_extension.rb +14 -1
  31. data/lib/datadog/profiling.rb +11 -0
  32. data/lib/datadog/tracing/sampling/matcher.rb +23 -3
  33. data/lib/datadog/tracing/sampling/rule.rb +7 -2
  34. data/lib/datadog/tracing/sampling/rule_sampler.rb +2 -0
  35. data/lib/ddtrace/version.rb +2 -2
  36. metadata +6 -17
  37. data/lib/datadog/core/telemetry/collector.rb +0 -250
  38. data/lib/datadog/core/telemetry/v1/app_event.rb +0 -59
  39. data/lib/datadog/core/telemetry/v1/application.rb +0 -92
  40. data/lib/datadog/core/telemetry/v1/configuration.rb +0 -25
  41. data/lib/datadog/core/telemetry/v1/dependency.rb +0 -43
  42. data/lib/datadog/core/telemetry/v1/host.rb +0 -59
  43. data/lib/datadog/core/telemetry/v1/install_signature.rb +0 -38
  44. data/lib/datadog/core/telemetry/v1/integration.rb +0 -64
  45. data/lib/datadog/core/telemetry/v1/product.rb +0 -36
  46. data/lib/datadog/core/telemetry/v1/telemetry_request.rb +0 -106
  47. data/lib/datadog/core/telemetry/v2/app_client_configuration_change.rb +0 -41
  48. data/lib/datadog/core/telemetry/v2/request.rb +0 -29
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 88e2b5d32d76e7c167d43a9868aed0903f8b82936bd0faf8eb10e02ae83d5928
4
- data.tar.gz: 65927b5d7037a853debb4da6cd71dda3dec0dd6b765bd90230ac3fa2904655f4
3
+ metadata.gz: 55da7739191e23ba6bb188ade9979c9dbd0a965bddfd2faedd6bf2c41b3a6e97
4
+ data.tar.gz: 98bfd95dc6dd156d3d57f2ac22deb081a8e6fb8d3221eadf6e682f45e358ef7c
5
5
  SHA512:
6
- metadata.gz: c467f5c384aeae1fe813df7a2b081e36c665f7a5f550322b9fbfafc402ad70d1d81ba578094bf4064a9d1c8a97f94be8b998ae2d2218c18e0bfaba97b754d5ea
7
- data.tar.gz: 8510f7f8afa6a89591bb128389ab210e2d6f3049c23a2d2ef3f58c4f0c042abe5cc865b6b5efe5198f116a6c98cea34cbf6e257078477744aa66db66aa3e6f1f
6
+ metadata.gz: 526f2826818f960ca18703a03cc0bde6313203a4e2dd03a696b8bd08f5a12cd963c370a8a9278d7387d81cf4445b849693fdbb8f91989f27f20d4f5bf55ad1a2
7
+ data.tar.gz: 9b8d42db31a997fd7f03e383e0e2671730577336f5bdb28bf36afd63e775639f584abb8d2005265835fc175c29ae275f5cdfea79ba9afe02eb16a3a826f47d70
data/CHANGELOG.md CHANGED
@@ -2,6 +2,33 @@
2
2
 
3
3
  ## [Unreleased]
4
4
 
5
+ ## [1.22.0] - 2024-04-16
6
+
7
+ ### Added
8
+
9
+ * Tracing: Add sampling rules by trace resouce and tags ([#3587][], [#3585][])
10
+ * Appsec: Add WAF vendor header support ([#3528][])
11
+
12
+ ### Changed
13
+
14
+ * Upgrade `Telemetry` to V2 ([#3551][])
15
+ * Upgrade to libdatadog 7 ([#3536][])
16
+ * Profiling: Enable Garbage Collection profiling by default ([#3558][])
17
+ * Profiling: Skip heap samples with age 0 ([#3573][])
18
+ * Profiling: Support falling back into extension directory when loading profiler ([#3582][])
19
+
20
+ ### Fixed
21
+
22
+ * Appsec: Fix MIME-style newlines with strict base64 encoding ([#3565][])
23
+
24
+ ## [2.0.0.beta1] - 2024-03-22
25
+
26
+ Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v2.0.0.beta1
27
+
28
+ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v1.21.1...v2.0.0.beta1
29
+
30
+ See https://github.com/DataDog/dd-trace-rb/blob/v2.0.0.beta1/docs/UpgradeGuide2.md.
31
+
5
32
  ## [1.21.1] - 2024-03-20
6
33
 
7
34
  ### Added
@@ -2776,7 +2803,9 @@ Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.3.1
2776
2803
  Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
2777
2804
 
2778
2805
 
2779
- [Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v1.21.1...master
2806
+ [Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v1.22.0...master
2807
+ [1.22.0]: https://github.com/DataDog/dd-trace-rb/compare/v1.21.1...v1.22.0
2808
+ [2.0.0.beta1]: https://github.com/DataDog/dd-trace-rb/compare/v1.21.1...v2.0.0.beta1
2780
2809
  [1.21.1]: https://github.com/DataDog/dd-trace-rb/compare/v1.21.0...v1.21.1
2781
2810
  [1.21.0]: https://github.com/DataDog/dd-trace-rb/compare/v1.20.0...v1.21.0
2782
2811
  [1.20.0]: https://github.com/DataDog/dd-trace-rb/compare/v1.19.0...v1.20.0
@@ -4070,9 +4099,18 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
4070
4099
  [#3519]: https://github.com/DataDog/dd-trace-rb/issues/3519
4071
4100
  [#3520]: https://github.com/DataDog/dd-trace-rb/issues/3520
4072
4101
  [#3523]: https://github.com/DataDog/dd-trace-rb/issues/3523
4102
+ [#3528]: https://github.com/DataDog/dd-trace-rb/issues/3528
4073
4103
  [#3531]: https://github.com/DataDog/dd-trace-rb/issues/3531
4074
4104
  [#3535]: https://github.com/DataDog/dd-trace-rb/issues/3535
4105
+ [#3536]: https://github.com/DataDog/dd-trace-rb/issues/3536
4075
4106
  [#3539]: https://github.com/DataDog/dd-trace-rb/issues/3539
4107
+ [#3551]: https://github.com/DataDog/dd-trace-rb/issues/3551
4108
+ [#3558]: https://github.com/DataDog/dd-trace-rb/issues/3558
4109
+ [#3565]: https://github.com/DataDog/dd-trace-rb/issues/3565
4110
+ [#3573]: https://github.com/DataDog/dd-trace-rb/issues/3573
4111
+ [#3582]: https://github.com/DataDog/dd-trace-rb/issues/3582
4112
+ [#3585]: https://github.com/DataDog/dd-trace-rb/issues/3585
4113
+ [#3587]: https://github.com/DataDog/dd-trace-rb/issues/3587
4076
4114
  [@AdrianLC]: https://github.com/AdrianLC
4077
4115
  [@Azure7111]: https://github.com/Azure7111
4078
4116
  [@BabyGroot]: https://github.com/BabyGroot
@@ -96,6 +96,7 @@ struct cpu_and_wall_time_worker_state {
96
96
  bool no_signals_workaround_enabled;
97
97
  bool dynamic_sampling_rate_enabled;
98
98
  bool allocation_profiling_enabled;
99
+ bool skip_idle_samples_for_testing;
99
100
  VALUE self_instance;
100
101
  VALUE thread_context_collector_instance;
101
102
  VALUE idle_sampling_helper_instance;
@@ -132,6 +133,8 @@ struct cpu_and_wall_time_worker_state {
132
133
  unsigned int signal_handler_enqueued_sample;
133
134
  // How many times the signal handler was called from the wrong thread
134
135
  unsigned int signal_handler_wrong_thread;
136
+ // How many times we actually tried to interrupt a thread for sampling
137
+ unsigned int interrupt_thread_attempts;
135
138
 
136
139
  // # Stats for the results of calling rb_postponed_job_register_one
137
140
  // The same function was already waiting to be executed
@@ -177,7 +180,8 @@ static VALUE _native_initialize(
177
180
  VALUE no_signals_workaround_enabled,
178
181
  VALUE dynamic_sampling_rate_enabled,
179
182
  VALUE dynamic_sampling_rate_overhead_target_percentage,
180
- VALUE allocation_profiling_enabled
183
+ VALUE allocation_profiling_enabled,
184
+ VALUE skip_idle_samples_for_testing
181
185
  );
182
186
  static void cpu_and_wall_time_worker_typed_data_mark(void *state_ptr);
183
187
  static VALUE _native_sampling_loop(VALUE self, VALUE instance);
@@ -272,14 +276,16 @@ void collectors_cpu_and_wall_time_worker_init(VALUE profiling_module) {
272
276
  // https://bugs.ruby-lang.org/issues/18007 for a discussion around this.
273
277
  rb_define_alloc_func(collectors_cpu_and_wall_time_worker_class, _native_new);
274
278
 
275
- rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_initialize", _native_initialize, 8);
279
+ rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_initialize", _native_initialize, 9);
276
280
  rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_sampling_loop", _native_sampling_loop, 1);
277
281
  rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_stop", _native_stop, 2);
278
282
  rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_reset_after_fork", _native_reset_after_fork, 1);
279
283
  rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_stats", _native_stats, 1);
280
284
  rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_stats_reset_not_thread_safe", _native_stats_reset_not_thread_safe, 1);
281
285
  rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_allocation_count", _native_allocation_count, 0);
286
+ rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_is_running?", _native_is_running, 1);
282
287
  rb_define_singleton_method(testing_module, "_native_current_sigprof_signal_handler", _native_current_sigprof_signal_handler, 0);
288
+ // TODO: Remove `_native_is_running` from `testing_module` once `prof-correctness` has been updated to not need it
283
289
  rb_define_singleton_method(testing_module, "_native_is_running?", _native_is_running, 1);
284
290
  rb_define_singleton_method(testing_module, "_native_install_testing_signal_handler", _native_install_testing_signal_handler, 0);
285
291
  rb_define_singleton_method(testing_module, "_native_remove_testing_signal_handler", _native_remove_testing_signal_handler, 0);
@@ -315,6 +321,7 @@ static VALUE _native_new(VALUE klass) {
315
321
  state->no_signals_workaround_enabled = false;
316
322
  state->dynamic_sampling_rate_enabled = true;
317
323
  state->allocation_profiling_enabled = false;
324
+ state->skip_idle_samples_for_testing = false;
318
325
  state->thread_context_collector_instance = Qnil;
319
326
  state->idle_sampling_helper_instance = Qnil;
320
327
  state->owner_thread = Qnil;
@@ -350,13 +357,15 @@ static VALUE _native_initialize(
350
357
  VALUE no_signals_workaround_enabled,
351
358
  VALUE dynamic_sampling_rate_enabled,
352
359
  VALUE dynamic_sampling_rate_overhead_target_percentage,
353
- VALUE allocation_profiling_enabled
360
+ VALUE allocation_profiling_enabled,
361
+ VALUE skip_idle_samples_for_testing
354
362
  ) {
355
363
  ENFORCE_BOOLEAN(gc_profiling_enabled);
356
364
  ENFORCE_BOOLEAN(no_signals_workaround_enabled);
357
365
  ENFORCE_BOOLEAN(dynamic_sampling_rate_enabled);
358
366
  ENFORCE_TYPE(dynamic_sampling_rate_overhead_target_percentage, T_FLOAT);
359
367
  ENFORCE_BOOLEAN(allocation_profiling_enabled);
368
+ ENFORCE_BOOLEAN(skip_idle_samples_for_testing)
360
369
 
361
370
  struct cpu_and_wall_time_worker_state *state;
362
371
  TypedData_Get_Struct(self_instance, struct cpu_and_wall_time_worker_state, &cpu_and_wall_time_worker_typed_data, state);
@@ -365,6 +374,7 @@ static VALUE _native_initialize(
365
374
  state->no_signals_workaround_enabled = (no_signals_workaround_enabled == Qtrue);
366
375
  state->dynamic_sampling_rate_enabled = (dynamic_sampling_rate_enabled == Qtrue);
367
376
  state->allocation_profiling_enabled = (allocation_profiling_enabled == Qtrue);
377
+ state->skip_idle_samples_for_testing = (skip_idle_samples_for_testing == Qtrue);
368
378
 
369
379
  double total_overhead_target_percentage = NUM2DBL(dynamic_sampling_rate_overhead_target_percentage);
370
380
  if (!state->allocation_profiling_enabled) {
@@ -616,17 +626,23 @@ static void *run_sampling_trigger_loop(void *state_ptr) {
616
626
  // Note that reading the GVL owner and sending them a signal is a race -- the Ruby VM keeps on executing while
617
627
  // we're doing this, so we may still not signal the correct thread from time to time, but our signal handler
618
628
  // includes a check to see if it got called in the right thread
629
+ state->stats.interrupt_thread_attempts++;
619
630
  pthread_kill(owner.owner, SIGPROF);
620
631
  } else {
621
- // If no thread owns the Global VM Lock, the application is probably idle at the moment. We still want to sample
622
- // so we "ask a friend" (the IdleSamplingHelper component) to grab the GVL and simulate getting a SIGPROF.
623
- //
624
- // In a previous version of the code, we called `grab_gvl_and_sample` directly BUT this was problematic because
625
- // Ruby may concurrently get busy and so the CpuAndWallTimeWorker would be blocked in line to acquire the GVL
626
- // for an uncontrolled amount of time. (This can still happen to the IdleSamplingHelper, but the
627
- // CpuAndWallTimeWorker will still be free to interrupt the Ruby VM and keep sampling for the entire blocking period).
628
- state->stats.trigger_simulated_signal_delivery_attempts++;
629
- idle_sampling_helper_request_action(state->idle_sampling_helper_instance, grab_gvl_and_sample);
632
+ if (state->skip_idle_samples_for_testing) {
633
+ // This was added to make sure our tests don't accidentally pass due to idle samples. Specifically, if we
634
+ // comment out the thread interruption code inside `if (owner.valid)` above, our tests should not pass!
635
+ } else {
636
+ // If no thread owns the Global VM Lock, the application is probably idle at the moment. We still want to sample
637
+ // so we "ask a friend" (the IdleSamplingHelper component) to grab the GVL and simulate getting a SIGPROF.
638
+ //
639
+ // In a previous version of the code, we called `grab_gvl_and_sample` directly BUT this was problematic because
640
+ // Ruby may concurrently get busy and so the CpuAndWallTimeWorker would be blocked in line to acquire the GVL
641
+ // for an uncontrolled amount of time. (This can still happen to the IdleSamplingHelper, but the
642
+ // CpuAndWallTimeWorker will still be free to interrupt the Ruby VM and keep sampling for the entire blocking period).
643
+ state->stats.trigger_simulated_signal_delivery_attempts++;
644
+ idle_sampling_helper_request_action(state->idle_sampling_helper_instance, grab_gvl_and_sample);
645
+ }
630
646
  }
631
647
  }
632
648
 
@@ -737,6 +753,9 @@ static VALUE release_gvl_and_run_sampling_trigger_loop(VALUE instance) {
737
753
  if (state->gc_profiling_enabled) rb_tracepoint_enable(state->gc_tracepoint);
738
754
  if (state->allocation_profiling_enabled) rb_tracepoint_enable(state->object_allocation_tracepoint);
739
755
 
756
+ // Flag the profiler as running before we release the GVL, in case anyone's waiting to know about it
757
+ rb_funcall(instance, rb_intern("signal_running"), 0);
758
+
740
759
  rb_thread_call_without_gvl(run_sampling_trigger_loop, state, interrupt_sampling_trigger_loop, state);
741
760
 
742
761
  // If we stopped sampling due to an exception, re-raise it (now in the worker thread)
@@ -943,6 +962,7 @@ static VALUE _native_stats(DDTRACE_UNUSED VALUE self, VALUE instance) {
943
962
  ID2SYM(rb_intern("postponed_job_success")), /* => */ UINT2NUM(state->stats.postponed_job_success),
944
963
  ID2SYM(rb_intern("postponed_job_full")), /* => */ UINT2NUM(state->stats.postponed_job_full),
945
964
  ID2SYM(rb_intern("postponed_job_unknown_result")), /* => */ UINT2NUM(state->stats.postponed_job_unknown_result),
965
+ ID2SYM(rb_intern("interrupt_thread_attempts")), /* => */ UINT2NUM(state->stats.interrupt_thread_attempts),
946
966
 
947
967
  // CPU Stats
948
968
  ID2SYM(rb_intern("cpu_sampled")), /* => */ UINT2NUM(state->stats.cpu_sampled),
@@ -621,11 +621,14 @@ bool thread_context_collector_on_gc_finish(VALUE self_instance) {
621
621
  // Let the caller know if it should schedule a flush or not. Returning true every time would cause a lot of overhead
622
622
  // on the application (see GC tracking introduction at the top of the file), so instead we try to accumulate a few
623
623
  // samples first.
624
- bool finished_major_gc = gc_profiling_has_major_gc_finished();
625
624
  bool over_flush_time_treshold =
626
625
  (wall_time_at_finish_ns - state->gc_tracking.wall_time_at_last_flushed_gc_event_ns) >= TIME_BETWEEN_GC_EVENTS_NS;
627
626
 
628
- return finished_major_gc || over_flush_time_treshold;
627
+ if (over_flush_time_treshold) {
628
+ return true;
629
+ } else {
630
+ return gc_profiling_has_major_gc_finished();
631
+ }
629
632
  }
630
633
 
631
634
  // This function gets called after one or more GC work steps (calls to on_gc_start/on_gc_finish).
@@ -10,6 +10,13 @@
10
10
  #define CAN_APPLY_GC_FORCE_RECYCLE_BUG_WORKAROUND
11
11
  #endif
12
12
 
13
+ // Minimum age (in GC generations) of heap objects we want to include in heap
14
+ // recorder iterations. Object with age 0 represent objects that have yet to undergo
15
+ // a GC and, thus, may just be noise/trash at instant of iteration and are usually not
16
+ // relevant for heap profiles as the great majority should be trivially reclaimed
17
+ // during the next GC.
18
+ #define ITERATION_MIN_AGE 1
19
+
13
20
  // A compact representation of a stacktrace frame for a heap allocation.
14
21
  typedef struct {
15
22
  char *name;
@@ -137,6 +144,11 @@ struct heap_recorder {
137
144
  // mutation of the data so iteration can occur without acquiring a lock.
138
145
  // NOTE: Contrary to object_records, this table has no ownership of its data.
139
146
  st_table *object_records_snapshot;
147
+ // The GC gen/epoch/count in which we prepared the current iteration.
148
+ //
149
+ // This enables us to calculate the age of iterated objects in the above snapshot by
150
+ // comparing it against an object's alloc_gen.
151
+ size_t iteration_gen;
140
152
 
141
153
  // Data for a heap recording that was started but not yet ended
142
154
  recording active_recording;
@@ -353,6 +365,8 @@ void heap_recorder_prepare_iteration(heap_recorder *heap_recorder) {
353
365
  return;
354
366
  }
355
367
 
368
+ heap_recorder->iteration_gen = rb_gc_count();
369
+
356
370
  if (heap_recorder->object_records_snapshot != NULL) {
357
371
  // we could trivially handle this but we raise to highlight and catch unexpected usages.
358
372
  rb_raise(rb_eRuntimeError, "New heap recorder iteration prepared without the previous one having been finished.");
@@ -459,6 +473,13 @@ static int st_object_record_entry_free(DDTRACE_UNUSED st_data_t key, st_data_t v
459
473
  return ST_DELETE;
460
474
  }
461
475
 
476
+ // Check to see if an object should not be included in a heap recorder iteration.
477
+ // This centralizes the checking logic to ensure it's equally applied between
478
+ // preparation and iteration codepaths.
479
+ static inline bool should_exclude_from_iteration(object_record *obj_record) {
480
+ return obj_record->object_data.gen_age < ITERATION_MIN_AGE;
481
+ }
482
+
462
483
  static int st_object_record_update(st_data_t key, st_data_t value, st_data_t extra_arg) {
463
484
  long obj_id = (long) key;
464
485
  object_record *record = (object_record*) value;
@@ -466,6 +487,19 @@ static int st_object_record_update(st_data_t key, st_data_t value, st_data_t ext
466
487
 
467
488
  VALUE ref;
468
489
 
490
+ size_t iteration_gen = recorder->iteration_gen;
491
+ size_t alloc_gen = record->object_data.alloc_gen;
492
+ // Guard against potential overflows given unsigned types here.
493
+ record->object_data.gen_age = alloc_gen < iteration_gen ? iteration_gen - alloc_gen : 0;
494
+
495
+ if (should_exclude_from_iteration(record)) {
496
+ // If an object won't be included in the current iteration, there's
497
+ // no point checking for liveness or updating its size, so exit early.
498
+ // NOTE: This means that there should be an equivalent check during actual
499
+ // iteration otherwise we'd iterate/expose stale object data.
500
+ return ST_CONTINUE;
501
+ }
502
+
469
503
  if (!ruby_ref_from_id(LONG2NUM(obj_id), &ref)) {
470
504
  // Id no longer associated with a valid ref. Need to delete this object record!
471
505
  on_committed_object_record_cleanup(recorder, record);
@@ -525,8 +559,16 @@ static int st_object_records_iterate(DDTRACE_UNUSED st_data_t key, st_data_t val
525
559
  const heap_stack *stack = record->heap_record->stack;
526
560
  iteration_context *context = (iteration_context*) extra;
527
561
 
528
- ddog_prof_Location *locations = context->heap_recorder->reusable_locations;
562
+ const heap_recorder *recorder = context->heap_recorder;
563
+
564
+ if (should_exclude_from_iteration(record)) {
565
+ // Skip objects that should not be included in iteration
566
+ // NOTE: This matches the short-circuiting condition in st_object_record_update
567
+ // and prevents iteration over stale objects.
568
+ return ST_CONTINUE;
569
+ }
529
570
 
571
+ ddog_prof_Location *locations = recorder->reusable_locations;
530
572
  for (uint16_t i = 0; i < stack->frames_len; i++) {
531
573
  const heap_frame *frame = &stack->frames[i];
532
574
  ddog_prof_Location *location = &locations[i];
@@ -725,9 +767,9 @@ void object_record_free(object_record *record) {
725
767
 
726
768
  VALUE object_record_inspect(object_record *record) {
727
769
  heap_frame top_frame = record->heap_record->stack->frames[0];
728
- VALUE inspect = rb_sprintf("obj_id=%ld weight=%d size=%zu location=%s:%d alloc_gen=%zu ",
770
+ VALUE inspect = rb_sprintf("obj_id=%ld weight=%d size=%zu location=%s:%d alloc_gen=%zu gen_age=%zu ",
729
771
  record->obj_id, record->object_data.weight, record->object_data.size, top_frame.filename,
730
- (int) top_frame.line, record->object_data.alloc_gen);
772
+ (int) top_frame.line, record->object_data.alloc_gen, record->object_data.gen_age);
731
773
 
732
774
  const char *class = record->object_data.class;
733
775
  if (class != NULL) {
@@ -27,7 +27,9 @@ typedef struct live_object_data {
27
27
  // could be seen as being representative of 50 objects.
28
28
  unsigned int weight;
29
29
 
30
- // Size of this object on last flush/update.
30
+ // Size of this object in memory.
31
+ // NOTE: This only gets updated during heap_recorder_prepare_iteration and only
32
+ // for those objects that meet the minimum iteration age requirements.
31
33
  size_t size;
32
34
 
33
35
  // The class of the object that we're tracking.
@@ -39,6 +41,10 @@ typedef struct live_object_data {
39
41
  // This enables us to calculate the age of this object in terms of GC executions.
40
42
  size_t alloc_gen;
41
43
 
44
+ // The age of this object in terms of GC generations.
45
+ // NOTE: This only gets updated during heap_recorder_prepare_iteration
46
+ size_t gen_age;
47
+
42
48
  // Whether this object was previously seen as being frozen. If this is the case,
43
49
  // we'll skip any further size updates since frozen objects are supposed to be
44
50
  // immutable.
@@ -30,7 +30,7 @@ inline static ddog_ByteSlice byte_slice_from_ruby_string(VALUE string);
30
30
  static VALUE _native_validate_exporter(VALUE self, VALUE exporter_configuration);
31
31
  static ddog_prof_Exporter_NewResult create_exporter(VALUE exporter_configuration, VALUE tags_as_array);
32
32
  static VALUE handle_exporter_failure(ddog_prof_Exporter_NewResult exporter_result);
33
- static ddog_Endpoint endpoint_from(VALUE exporter_configuration);
33
+ static ddog_prof_Endpoint endpoint_from(VALUE exporter_configuration);
34
34
  static ddog_Vec_Tag convert_tags(VALUE tags_as_array);
35
35
  static void safely_log_failure_to_process_tag(ddog_Vec_Tag tags, VALUE err_details);
36
36
  static VALUE _native_do_export(
@@ -94,7 +94,7 @@ static ddog_prof_Exporter_NewResult create_exporter(VALUE exporter_configuration
94
94
 
95
95
  // This needs to be called BEFORE convert_tags since it can raise an exception and thus cause the ddog_Vec_Tag
96
96
  // to be leaked.
97
- ddog_Endpoint endpoint = endpoint_from(exporter_configuration);
97
+ ddog_prof_Endpoint endpoint = endpoint_from(exporter_configuration);
98
98
 
99
99
  ddog_Vec_Tag tags = convert_tags(tags_as_array);
100
100
 
@@ -116,7 +116,7 @@ static VALUE handle_exporter_failure(ddog_prof_Exporter_NewResult exporter_resul
116
116
  rb_ary_new_from_args(2, error_symbol, get_error_details_and_drop(&exporter_result.err));
117
117
  }
118
118
 
119
- static ddog_Endpoint endpoint_from(VALUE exporter_configuration) {
119
+ static ddog_prof_Endpoint endpoint_from(VALUE exporter_configuration) {
120
120
  ENFORCE_TYPE(exporter_configuration, T_ARRAY);
121
121
 
122
122
  ID working_mode = SYM2ID(rb_ary_entry(exporter_configuration, 0)); // SYM2ID verifies its input so we can do this safely
@@ -131,12 +131,12 @@ static ddog_Endpoint endpoint_from(VALUE exporter_configuration) {
131
131
  ENFORCE_TYPE(site, T_STRING);
132
132
  ENFORCE_TYPE(api_key, T_STRING);
133
133
 
134
- return ddog_Endpoint_agentless(char_slice_from_ruby_string(site), char_slice_from_ruby_string(api_key));
134
+ return ddog_prof_Endpoint_agentless(char_slice_from_ruby_string(site), char_slice_from_ruby_string(api_key));
135
135
  } else { // agent_id
136
136
  VALUE base_url = rb_ary_entry(exporter_configuration, 1);
137
137
  ENFORCE_TYPE(base_url, T_STRING);
138
138
 
139
- return ddog_Endpoint_agent(char_slice_from_ruby_string(base_url));
139
+ return ddog_prof_Endpoint_agent(char_slice_from_ruby_string(base_url));
140
140
  }
141
141
  }
142
142
 
@@ -15,7 +15,7 @@ module Datadog
15
15
  # The MJIT header was introduced on 2.6 and removed on 3.3; for other Rubies we rely on debase-ruby_core_source
16
16
  CAN_USE_MJIT_HEADER = RUBY_VERSION.start_with?('2.6', '2.7', '3.0.', '3.1.', '3.2.')
17
17
 
18
- LIBDATADOG_VERSION = '~> 6.0.0.2.0'
18
+ LIBDATADOG_VERSION = '~> 7.0.0.1.0'
19
19
 
20
20
  def self.fail_install_if_missing_extension?
21
21
  ENV[ENV_FAIL_INSTALL_IF_MISSING_EXTENSION].to_s.strip.downcase == 'true'
@@ -196,7 +196,6 @@ struct call_serialize_without_gvl_arguments {
196
196
  // Set by caller
197
197
  struct stack_recorder_state *state;
198
198
  ddog_Timespec finish_timestamp;
199
- size_t gc_count_before_serialize;
200
199
 
201
200
  // Set by callee
202
201
  ddog_prof_Profile *profile;
@@ -489,7 +488,6 @@ static VALUE _native_serialize(DDTRACE_UNUSED VALUE _self, VALUE recorder_instan
489
488
  struct call_serialize_without_gvl_arguments args = {
490
489
  .state = state,
491
490
  .finish_timestamp = finish_timestamp,
492
- .gc_count_before_serialize = rb_gc_count(),
493
491
  .serialize_ran = false
494
492
  };
495
493
 
@@ -613,8 +611,6 @@ typedef struct heap_recorder_iteration_context {
613
611
 
614
612
  bool error;
615
613
  char error_msg[MAX_LEN_HEAP_ITERATION_ERROR_MSG];
616
-
617
- size_t profile_gen;
618
614
  } heap_recorder_iteration_context;
619
615
 
620
616
  static bool add_heap_sample_to_active_profile_without_gvl(heap_recorder_iteration_data iteration_data, void *extra_arg) {
@@ -643,7 +639,7 @@ static bool add_heap_sample_to_active_profile_without_gvl(heap_recorder_iteratio
643
639
  }
644
640
  labels[label_offset++] = (ddog_prof_Label) {
645
641
  .key = DDOG_CHARSLICE_C("gc gen age"),
646
- .num = context->profile_gen - object_data->alloc_gen,
642
+ .num = object_data->gen_age,
647
643
  };
648
644
 
649
645
  ddog_prof_Profile_Result result = ddog_prof_Profile_add(
@@ -670,13 +666,12 @@ static bool add_heap_sample_to_active_profile_without_gvl(heap_recorder_iteratio
670
666
  return true;
671
667
  }
672
668
 
673
- static void build_heap_profile_without_gvl(struct stack_recorder_state *state, ddog_prof_Profile *profile, size_t gc_count_before_serialize) {
669
+ static void build_heap_profile_without_gvl(struct stack_recorder_state *state, ddog_prof_Profile *profile) {
674
670
  heap_recorder_iteration_context iteration_context = {
675
671
  .state = state,
676
672
  .profile = profile,
677
673
  .error = false,
678
674
  .error_msg = {0},
679
- .profile_gen = gc_count_before_serialize,
680
675
  };
681
676
  bool iterated = heap_recorder_for_each_live_object(state->heap_recorder, add_heap_sample_to_active_profile_without_gvl, (void*) &iteration_context);
682
677
  // We wait until we're out of the iteration to grab the gvl and raise. This is important because during
@@ -698,7 +693,7 @@ static void *call_serialize_without_gvl(void *call_args) {
698
693
 
699
694
  // Now that we have the inactive profile with all but heap samples, lets fill it with heap data
700
695
  // without needing to race with the active sampler
701
- build_heap_profile_without_gvl(args->state, args->profile, args->gc_count_before_serialize);
696
+ build_heap_profile_without_gvl(args->state, args->profile);
702
697
 
703
698
  // Note: The profile gets reset by the serialize call
704
699
  args->result = ddog_prof_Profile_serialize(args->profile, &args->finish_timestamp, NULL /* duration_nanos is optional */, NULL /* start_time is optional */);
@@ -918,10 +913,13 @@ static VALUE _native_debug_heap_recorder(DDTRACE_UNUSED VALUE _self, VALUE recor
918
913
  #pragma GCC diagnostic push
919
914
  // rb_gc_force_recycle was deprecated in latest versions of Ruby and is a noop.
920
915
  #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
916
+ #pragma GCC diagnostic ignored "-Wunused-parameter"
921
917
  // This method exists only to enable testing Datadog::Profiling::StackRecorder behavior using RSpec.
922
918
  // It SHOULD NOT be used for other purposes.
923
919
  static VALUE _native_gc_force_recycle(DDTRACE_UNUSED VALUE _self, VALUE obj) {
924
- rb_gc_force_recycle(obj);
920
+ #ifdef HAVE_WORKING_RB_GC_FORCE_RECYCLE
921
+ rb_gc_force_recycle(obj);
922
+ #endif
925
923
  return Qnil;
926
924
  }
927
925
  #pragma GCC diagnostic pop
@@ -13,6 +13,18 @@ module Datadog
13
13
  module AppSec
14
14
  module Contrib
15
15
  module Rack
16
+ # Create an array of lowercased headers
17
+ WAF_VENDOR_HEADERS_TAGS = %w[
18
+ X-Amzn-Trace-Id
19
+ Cloudfront-Viewer-Ja3-Fingerprint
20
+ Cf-Ray
21
+ X-Cloud-Trace-Context
22
+ X-Appgw-Trace-id
23
+ X-SigSci-RequestID
24
+ X-SigSci-Tags
25
+ Akamai-User-Risk
26
+ ].map(&:downcase).freeze
27
+
16
28
  # Topmost Rack middleware for AppSec
17
29
  # This should be inserted just below Datadog::Tracing::Contrib::Rack::TraceMiddleware
18
30
  class RequestMiddleware
@@ -20,6 +32,7 @@ module Datadog
20
32
  @app = app
21
33
 
22
34
  @oneshot_tags_sent = false
35
+ @rack_headers = {}
23
36
  end
24
37
 
25
38
  # rubocop:disable Metrics/AbcSize,Metrics/PerceivedComplexity,Metrics/CyclomaticComplexity,Metrics/MethodLength
@@ -54,7 +67,8 @@ module Datadog
54
67
 
55
68
  gateway_request = Gateway::Request.new(env)
56
69
 
57
- add_appsec_tags(processor, scope, env)
70
+ add_appsec_tags(processor, scope)
71
+ add_request_tags(scope, env)
58
72
 
59
73
  request_return, request_response = catch(::Datadog::AppSec::Ext::INTERRUPT) do
60
74
  Instrumentation.gateway.push('rack.request', gateway_request) do
@@ -129,7 +143,7 @@ module Datadog
129
143
  Datadog::Tracing.active_span
130
144
  end
131
145
 
132
- def add_appsec_tags(processor, scope, env)
146
+ def add_appsec_tags(processor, scope)
133
147
  span = scope.service_entry_span
134
148
  trace = scope.trace
135
149
 
@@ -139,17 +153,6 @@ module Datadog
139
153
  span.set_tag('_dd.runtime_family', 'ruby')
140
154
  span.set_tag('_dd.appsec.waf.version', Datadog::AppSec::WAF::VERSION::BASE_STRING)
141
155
 
142
- if span && span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
143
- request_header_collection = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
144
-
145
- # always collect client ip, as this is part of AppSec provided functionality
146
- Datadog::Tracing::ClientIp.set_client_ip_tag!(
147
- span,
148
- headers: request_header_collection,
149
- remote_ip: env['REMOTE_ADDR']
150
- )
151
- end
152
-
153
156
  if processor.diagnostics
154
157
  diagnostics = processor.diagnostics
155
158
 
@@ -175,6 +178,29 @@ module Datadog
175
178
  end
176
179
  end
177
180
 
181
+ def add_request_tags(scope, env)
182
+ span = scope.service_entry_span
183
+
184
+ return unless span
185
+
186
+ # Always add WAF vendors headers
187
+ WAF_VENDOR_HEADERS_TAGS.each do |lowercase_header|
188
+ rack_header = to_rack_header(lowercase_header)
189
+ span.set_tag("http.request.headers.#{lowercase_header}", env[rack_header]) if env[rack_header]
190
+ end
191
+
192
+ if span && span.get_tag(Tracing::Metadata::Ext::HTTP::TAG_CLIENT_IP).nil?
193
+ request_header_collection = Datadog::Tracing::Contrib::Rack::Header::RequestHeaderCollection.new(env)
194
+
195
+ # always collect client ip, as this is part of AppSec provided functionality
196
+ Datadog::Tracing::ClientIp.set_client_ip_tag!(
197
+ span,
198
+ headers: request_header_collection,
199
+ remote_ip: env['REMOTE_ADDR']
200
+ )
201
+ end
202
+ end
203
+
178
204
  def add_waf_runtime_tags(scope)
179
205
  span = scope.service_entry_span
180
206
  context = scope.processor_context
@@ -187,6 +213,10 @@ module Datadog
187
213
  span.set_tag('_dd.appsec.waf.duration', context.time_ns / 1000.0)
188
214
  span.set_tag('_dd.appsec.waf.duration_ext', context.time_ext_ns / 1000.0)
189
215
  end
216
+
217
+ def to_rack_header(header)
218
+ @rack_headers[header] ||= Datadog::Tracing::Contrib::Rack::Header.to_rack_header(header)
219
+ end
190
220
  end
191
221
  end
192
222
  end
@@ -138,7 +138,7 @@ module Datadog
138
138
  private
139
139
 
140
140
  def compressed_and_base64_encoded(value)
141
- Base64.encode64(gzip(value))
141
+ Base64.strict_encode64(gzip(value))
142
142
  rescue TypeError => e
143
143
  Datadog.logger.debug do
144
144
  "Failed to compress and encode value when populating AppSec::Event. Error: #{e.message}"
@@ -62,7 +62,8 @@ module Datadog
62
62
 
63
63
  Telemetry::Client.new(
64
64
  enabled: enabled,
65
- heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds
65
+ heartbeat_interval_seconds: settings.telemetry.heartbeat_interval_seconds,
66
+ dependency_collection: settings.telemetry.dependency_collection
66
67
  )
67
68
  end
68
69
  end
@@ -8,7 +8,13 @@ module Datadog
8
8
  # Represents an instance of an integration configuration option
9
9
  # @public_api
10
10
  class Option
11
- attr_reader :definition
11
+ # @!attribute [r] definition
12
+ # The definition object that matches this option.
13
+ # @return [Configuration::OptionDefinition]
14
+ # @!attribute [r] precedence_set
15
+ # When this option was last set, what was the value precedence used?
16
+ # @return [Precedence::Value]
17
+ attr_reader :definition, :precedence_set
12
18
 
13
19
  # Option setting precedence.
14
20
  module Precedence
@@ -303,10 +309,6 @@ module Datadog
303
309
  ['true', '1'].include?(ENV.fetch('DD_EXPERIMENTAL_SKIP_CONFIGURATION_VALIDATION', '').strip)
304
310
  end
305
311
 
306
- # Used for testing
307
- attr_reader :precedence_set
308
- private :precedence_set
309
-
310
312
  # Anchor object that represents a value that is not set.
311
313
  # This is necessary because `nil` is a valid value to be set.
312
314
  UNSET = Object.new