ddtrace 1.10.1 → 1.11.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 005fcbaaa9553287f8652a9f437abd144c2ba2ca0f26c83f67de171646f5aae3
-  data.tar.gz: 032c0d3f09ea01779b17ba45a95086ab890b3869208c84b82b548b3cbd41a977
+  metadata.gz: 83276ed5293cf8971f19f7a1fb1198e01f5ff8d48d3dd440deff0ef3518939d6
+  data.tar.gz: 12d22b6020e8830c91b2f68aed9ce03a2d201eafc4911c0a95171df6480cf754
 SHA512:
-  metadata.gz: bc868d2e56e5282771cbc7e9447f1fa7504212d31b5bc94db4db9e5f029a00b0a9f988bcab7111f1889eb1f203fffcd81b73700c69f433be4bdaa2045dbfba72
-  data.tar.gz: d2679f725e7991bfa2e34dc6de650e928a59aa02124239d340a396ea40f331ff0a4a2159067df5ade22c1c36daab8e7cbe25db3ddaf55b8c8cb14137e7b2d220
+  metadata.gz: 4d4f7fb37bd10399053c026ff92545c5a1d600447cb94b3b35439fb37008154687c09f1834115afe4ed5cddc55fe84b458db955fcd4dbd0140444ffdaa01f8a9
+  data.tar.gz: 39d5ac62e456654550f70989e18dd3b122d3167aefc4a6b55ed5dc2ffd22c8f2b51c2f8261b2e4c009310acc2001be1a8f6ab6b0651850a45cc49cf0319ba6b1

data/CHANGELOG.md

@@ -2,6 +2,39 @@
 
 ## [Unreleased]
 
+## [1.11.0.beta1] - 2023-04-14
+
+As of ddtrace 1.11.0.beta1, CPU Profiling 2.0 is now GA and enabled by default. For more details, check the release notes.
+
+As of ddtrace 1.11.0.beta1, Remote Configuration is now public beta and disabled by default. For more details, check the release notes.
+
+### Added
+
+* Add remote configuration beta, disabled by default ([#2674][], [#2678][], [#2686][], [#2687][], [#2688][], [#2689][], [#2696][], [#2705][], [#2731][], [#2732][], [#2733][], [#2739][], [#2756][], [#2769][], [#2771][], [#2773][], [#2789][])
+* AppSec: Add response headers passing to WAF ([#2701][])
+* Tracing: Distributed tracing for Sidekiq ([#2513][])
+* Tracing: Add Roda integration ([#2368][])
+* Profiling: [PROF-6555] Support disabling endpoint names collection in new profiler ([#2698][])
+
+### Changed
+
+* Core: Allow `1` as true value in environment variables ([#2710][])
+* Profiling: [PROF-7360] Enable CPU Profiling 2.0 by default ([#2702][])
+* Tracing: Improve controller instrumentation and deprecate option `exception_controller` ([#2726][])
+* Tracing: Implement Span Attribute Schema Environment Variable ([#2727][])
+
+### Fixed
+
+* Bug: Tracing: Fix w3c propagation special character handling ([#2720][])
+* Performance: Tracing: Use `+@` instead of `dup` for duplicating strings ([#2704][])
+* Profiling: [PROF-7307] Avoid triggering allocation sampling during sampling ([#2690][])
+* Integrations: Tracing: Fix Rails < 3 conditional check in Utils#railtie_supported? ([#2695][])
+* Profiling: [PROF-7409] Do not auto-enable new profiler when rugged gem is detected ([#2741][])
+* Tracing: Fix using SemanticLogger#log(severity, message, progname) ([#2748][]) ([@rqz13][])
+* Profiling: [PROF-6447] Improve detection of mysql2 gem incompatibilities with profiler ([#2770][])
+* AppSec: Remove check for `::Rack::Request.instance_methods.include?(:each_header)` at load time ([#2778][])
+* Tracing: Fix quadratic backtracking on invalid URI ([#2788][])
+
 ## [1.10.1] - 2023-03-10
 
 ### Fixed
@@ -2323,7 +2356,8 @@ Release notes: https://github.com/DataDog/dd-trace-rb/releases/tag/v0.3.1
 
 Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 
-[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v1.10.1...master
+[Unreleased]: https://github.com/DataDog/dd-trace-rb/compare/v1.11.0.beta1...master
+[1.11.0.beta1]: https://github.com/DataDog/dd-trace-rb/compare/v1.10.1...v1.11.0.beta1
 [1.10.1]: https://github.com/DataDog/dd-trace-rb/compare/v1.10.0...v1.10.1
 [1.10.0]: https://github.com/DataDog/dd-trace-rb/compare/v1.9.0...v1.10.0
 [1.9.0]: https://github.com/DataDog/dd-trace-rb/compare/v1.8.0...v1.9.0
@@ -3261,6 +3295,7 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#2363]: https://github.com/DataDog/dd-trace-rb/issues/2363
 [#2365]: https://github.com/DataDog/dd-trace-rb/issues/2365
 [#2367]: https://github.com/DataDog/dd-trace-rb/issues/2367
+[#2368]: https://github.com/DataDog/dd-trace-rb/issues/2368
 [#2378]: https://github.com/DataDog/dd-trace-rb/issues/2378
 [#2382]: https://github.com/DataDog/dd-trace-rb/issues/2382
 [#2390]: https://github.com/DataDog/dd-trace-rb/issues/2390
@@ -3293,6 +3328,7 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#2501]: https://github.com/DataDog/dd-trace-rb/issues/2501
 [#2504]: https://github.com/DataDog/dd-trace-rb/issues/2504
 [#2512]: https://github.com/DataDog/dd-trace-rb/issues/2512
+[#2513]: https://github.com/DataDog/dd-trace-rb/issues/2513
 [#2522]: https://github.com/DataDog/dd-trace-rb/issues/2522
 [#2526]: https://github.com/DataDog/dd-trace-rb/issues/2526
 [#2530]: https://github.com/DataDog/dd-trace-rb/issues/2530
@@ -3334,7 +3370,39 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [#2663]: https://github.com/DataDog/dd-trace-rb/issues/2663
 [#2665]: https://github.com/DataDog/dd-trace-rb/issues/2665
 [#2668]: https://github.com/DataDog/dd-trace-rb/issues/2668
+[#2674]: https://github.com/DataDog/dd-trace-rb/issues/2674
+[#2678]: https://github.com/DataDog/dd-trace-rb/issues/2678
 [#2679]: https://github.com/DataDog/dd-trace-rb/issues/2679
+[#2686]: https://github.com/DataDog/dd-trace-rb/issues/2686
+[#2687]: https://github.com/DataDog/dd-trace-rb/issues/2687
+[#2688]: https://github.com/DataDog/dd-trace-rb/issues/2688
+[#2689]: https://github.com/DataDog/dd-trace-rb/issues/2689
+[#2690]: https://github.com/DataDog/dd-trace-rb/issues/2690
+[#2695]: https://github.com/DataDog/dd-trace-rb/issues/2695
+[#2696]: https://github.com/DataDog/dd-trace-rb/issues/2696
+[#2698]: https://github.com/DataDog/dd-trace-rb/issues/2698
+[#2701]: https://github.com/DataDog/dd-trace-rb/issues/2701
+[#2702]: https://github.com/DataDog/dd-trace-rb/issues/2702
+[#2704]: https://github.com/DataDog/dd-trace-rb/issues/2704
+[#2705]: https://github.com/DataDog/dd-trace-rb/issues/2705
+[#2710]: https://github.com/DataDog/dd-trace-rb/issues/2710
+[#2720]: https://github.com/DataDog/dd-trace-rb/issues/2720
+[#2726]: https://github.com/DataDog/dd-trace-rb/issues/2726
+[#2727]: https://github.com/DataDog/dd-trace-rb/issues/2727
+[#2731]: https://github.com/DataDog/dd-trace-rb/issues/2731
+[#2732]: https://github.com/DataDog/dd-trace-rb/issues/2732
+[#2733]: https://github.com/DataDog/dd-trace-rb/issues/2733
+[#2739]: https://github.com/DataDog/dd-trace-rb/issues/2739
+[#2741]: https://github.com/DataDog/dd-trace-rb/issues/2741
+[#2748]: https://github.com/DataDog/dd-trace-rb/issues/2748
+[#2756]: https://github.com/DataDog/dd-trace-rb/issues/2756
+[#2769]: https://github.com/DataDog/dd-trace-rb/issues/2769
+[#2770]: https://github.com/DataDog/dd-trace-rb/issues/2770
+[#2771]: https://github.com/DataDog/dd-trace-rb/issues/2771
+[#2773]: https://github.com/DataDog/dd-trace-rb/issues/2773
+[#2778]: https://github.com/DataDog/dd-trace-rb/issues/2778
+[#2788]: https://github.com/DataDog/dd-trace-rb/issues/2788
+[#2789]: https://github.com/DataDog/dd-trace-rb/issues/2789
 [@AdrianLC]: https://github.com/AdrianLC
 [@Azure7111]: https://github.com/Azure7111
 [@BabyGroot]: https://github.com/BabyGroot
@@ -3456,6 +3524,7 @@ Git diff: https://github.com/DataDog/dd-trace-rb/compare/v0.3.0...v0.3.1
 [@renchap]: https://github.com/renchap
 [@ricbartm]: https://github.com/ricbartm
 [@roccoblues]: https://github.com/roccoblues
+[@rqz13]: https://github.com/rqz13
 [@saturnflyer]: https://github.com/saturnflyer
 [@sco11morgan]: https://github.com/sco11morgan
 [@senny]: https://github.com/senny

data/README.md

@@ -3,27 +3,26 @@
 [![Gem](https://img.shields.io/gem/v/ddtrace)](https://rubygems.org/gems/ddtrace/)
 [![CircleCI](https://circleci.com/gh/DataDog/dd-trace-rb/tree/master.svg?style=svg&circle-token=b0bd5ef866ec7f7b018f48731bb495f2d1372cc1)](https://circleci.com/gh/DataDog/dd-trace-rb/tree/master)
 [![codecov](https://codecov.io/gh/DataDog/dd-trace-rb/branch/master/graph/badge.svg)](https://app.codecov.io/gh/DataDog/dd-trace-rb/branch/master)
-[![YARD documentation](https://img.shields.io/badge/YARD-documentation-blue)](https://s3.amazonaws.com/gems.datadoghq.com/trace/docs/index.html)
+[![YARD documentation](https://img.shields.io/badge/YARD-documentation-blue)][api docs]
 
 ``ddtrace`` is Datadog’s tracing client for Ruby. It is used to trace requests as they flow across web servers,
-databases and microservices so that developers have great visiblity into bottlenecks and troublesome requests.
+databases and microservices so that developers have great visibility into bottlenecks and troublesome requests.
 
 ## Getting started
 
 **If you're upgrading from a 0.x version, check out our [upgrade guide](https://github.com/DataDog/dd-trace-rb/blob/master/docs/UpgradeGuide.md#from-0x-to-10).**
 
-For a basic product overview, check out our [setup documentation][setup docs].
+For a product overview, installation, and configuration check out our [documentation][public docs].
 
-For installation, configuration, and details about using the API, check out our [API documentation][api docs] and [gem documentation][gem docs].
+For the gem API, check out our [API documentation][api docs].
 
-For descriptions of terminology used in APM, take a look at the [official documentation][visualization docs].
+For descriptions of terminology used in APM, take a look at the [APM Terms and Concepts][APM glossary].
 
 For contributing, checkout the [contribution guidelines][contribution docs] and [development guide][development docs].
 
-[setup docs]: https://docs.datadoghq.com/tracing/setup/ruby/
-[api docs]: https://github.com/DataDog/dd-trace-rb/blob/master/docs/GettingStarted.md
-[gem docs]: https://s3.amazonaws.com/gems.datadoghq.com/trace/docs/index.html
-[visualization docs]: https://docs.datadoghq.com/tracing/visualization/
+[public docs]: https://docs.datadoghq.com/tracing/setup/ruby/
+[api docs]: https://datadog.github.io/dd-trace-rb/
+[APM glossary]: https://docs.datadoghq.com/tracing/glossary/
 [contribution docs]: https://github.com/DataDog/dd-trace-rb/blob/master/CONTRIBUTING.md
 [development docs]: https://github.com/DataDog/dd-trace-rb/blob/master/docs/DevelopmentGuide.md
 

data/ext/ddtrace_profiling_native_extension/collectors_cpu_and_wall_time_worker.c

@@ -77,26 +77,33 @@
 
 // Contains state for a single CpuAndWallTimeWorker instance
 struct cpu_and_wall_time_worker_state {
-  atomic_bool should_run;
+  // These are immutable after initialization
 
   bool gc_profiling_enabled;
   bool allocation_counting_enabled;
+  bool dynamic_sampling_rate_enabled;
   VALUE self_instance;
   VALUE thread_context_collector_instance;
   VALUE idle_sampling_helper_instance;
   VALUE owner_thread;
   dynamic_sampling_rate_state dynamic_sampling_rate;
+  VALUE gc_tracepoint; // Used to get gc start/finish information
+  VALUE object_allocation_tracepoint; // Used to get allocation counts and allocation profiling
+
+  // These are mutable and used to signal things between the worker thread and other threads
 
+  atomic_bool should_run;
   // When something goes wrong during sampling, we record the Ruby exception here, so that it can be "re-raised" on
   // the CpuAndWallTimeWorker thread
   VALUE failure_exception;
   // Used by `_native_stop` to flag the worker thread to start (see comment on `_native_sampling_loop`)
   VALUE stop_thread;
 
-  // Used to get gc start/finish information
-  VALUE gc_tracepoint;
+  // Others
 
-  VALUE object_allocation_tracepoint;
+  // Used to detect/avoid nested sampling, e.g. when the object_allocation_tracepoint gets triggered by a memory allocation
+  // that happens during another sample.
+  bool during_sample;
 
   struct stats {
     // How many times we tried to trigger a sample
@@ -115,6 +122,8 @@ struct cpu_and_wall_time_worker_state {
     uint64_t sampling_time_ns_min;
     uint64_t sampling_time_ns_max;
     uint64_t sampling_time_ns_total;
+    // How many times we saw allocations being done inside a sample
+    unsigned int allocations_during_sample;
   } stats;
 };
 
@@ -125,7 +134,8 @@ static VALUE _native_initialize(
   VALUE thread_context_collector_instance,
   VALUE gc_profiling_enabled,
   VALUE idle_sampling_helper_instance,
-  VALUE allocation_counting_enabled
+  VALUE allocation_counting_enabled,
+  VALUE dynamic_sampling_rate_enabled
 );
 static void cpu_and_wall_time_worker_typed_data_mark(void *state_ptr);
 static VALUE _native_sampling_loop(VALUE self, VALUE instance);
@@ -160,6 +170,7 @@ static void sleep_for(uint64_t time_ns);
 static VALUE _native_allocation_count(DDTRACE_UNUSED VALUE self);
 static void on_newobj_event(DDTRACE_UNUSED VALUE tracepoint_data, DDTRACE_UNUSED void *unused);
 static void disable_tracepoints(struct cpu_and_wall_time_worker_state *state);
+static VALUE _native_with_blocked_sigprof(DDTRACE_UNUSED VALUE self);
 
 // Note on sampler global state safety:
 //
@@ -197,7 +208,7 @@ void collectors_cpu_and_wall_time_worker_init(VALUE profiling_module) {
   // https://bugs.ruby-lang.org/issues/18007 for a discussion around this.
   rb_define_alloc_func(collectors_cpu_and_wall_time_worker_class, _native_new);
 
-  rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_initialize", _native_initialize, 5);
+  rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_initialize", _native_initialize, 6);
   rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_sampling_loop", _native_sampling_loop, 1);
   rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_stop", _native_stop, 2);
   rb_define_singleton_method(collectors_cpu_and_wall_time_worker_class, "_native_reset_after_fork", _native_reset_after_fork, 1);
@@ -212,6 +223,7 @@ void collectors_cpu_and_wall_time_worker_init(VALUE profiling_module) {
   rb_define_singleton_method(testing_module, "_native_simulate_handle_sampling_signal", _native_simulate_handle_sampling_signal, 0);
   rb_define_singleton_method(testing_module, "_native_simulate_sample_from_postponed_job", _native_simulate_sample_from_postponed_job, 0);
   rb_define_singleton_method(testing_module, "_native_is_sigprof_blocked_in_current_thread", _native_is_sigprof_blocked_in_current_thread, 0);
+  rb_define_singleton_method(testing_module, "_native_with_blocked_sigprof", _native_with_blocked_sigprof, 0);
 }
 
 // This structure is used to define a Ruby object that stores a pointer to a struct cpu_and_wall_time_worker_state
@@ -230,17 +242,22 @@ static const rb_data_type_t cpu_and_wall_time_worker_typed_data = {
 static VALUE _native_new(VALUE klass) {
   struct cpu_and_wall_time_worker_state *state = ruby_xcalloc(1, sizeof(struct cpu_and_wall_time_worker_state));
 
-  atomic_init(&state->should_run, false);
   state->gc_profiling_enabled = false;
   state->allocation_counting_enabled = false;
+  state->dynamic_sampling_rate_enabled = true;
   state->thread_context_collector_instance = Qnil;
   state->idle_sampling_helper_instance = Qnil;
   state->owner_thread = Qnil;
   dynamic_sampling_rate_init(&state->dynamic_sampling_rate);
-  state->failure_exception = Qnil;
-  state->stop_thread = Qnil;
   state->gc_tracepoint = Qnil;
   state->object_allocation_tracepoint = Qnil;
+
+  atomic_init(&state->should_run, false);
+  state->failure_exception = Qnil;
+  state->stop_thread = Qnil;
+
+  state->during_sample = false;
+
   reset_stats(state);
 
   return state->self_instance = TypedData_Wrap_Struct(klass, &cpu_and_wall_time_worker_typed_data, state);
@@ -252,16 +269,19 @@ static VALUE _native_initialize(
   VALUE thread_context_collector_instance,
   VALUE gc_profiling_enabled,
   VALUE idle_sampling_helper_instance,
-  VALUE allocation_counting_enabled
+  VALUE allocation_counting_enabled,
+  VALUE dynamic_sampling_rate_enabled
 ) {
   ENFORCE_BOOLEAN(gc_profiling_enabled);
   ENFORCE_BOOLEAN(allocation_counting_enabled);
+  ENFORCE_BOOLEAN(dynamic_sampling_rate_enabled);
 
   struct cpu_and_wall_time_worker_state *state;
   TypedData_Get_Struct(self_instance, struct cpu_and_wall_time_worker_state, &cpu_and_wall_time_worker_typed_data, state);
 
   state->gc_profiling_enabled = (gc_profiling_enabled == Qtrue);
   state->allocation_counting_enabled = (allocation_counting_enabled == Qtrue);
+  state->dynamic_sampling_rate_enabled = (dynamic_sampling_rate_enabled == Qtrue);
   state->thread_context_collector_instance = enforce_thread_context_collector_instance(thread_context_collector_instance);
   state->idle_sampling_helper_instance = idle_sampling_helper_instance;
   state->gc_tracepoint = rb_tracepoint_new(Qnil, RUBY_INTERNAL_EVENT_GC_ENTER | RUBY_INTERNAL_EVENT_GC_EXIT, on_gc_event, NULL /* unused */);
@@ -460,7 +480,7 @@ static void *run_sampling_trigger_loop(void *state_ptr) {
     // `dynamic_sampling_rate_get_sleep` may have changed while the above sleep was ongoing.
     uint64_t extra_sleep =
       dynamic_sampling_rate_get_sleep(&state->dynamic_sampling_rate, monotonic_wall_time_now_ns(DO_NOT_RAISE_ON_FAILURE));
-    if (extra_sleep > 0) sleep_for(extra_sleep);
+    if (state->dynamic_sampling_rate_enabled && extra_sleep > 0) sleep_for(extra_sleep);
   }
 
   return NULL; // Unused
@@ -485,8 +505,12 @@ static void sample_from_postponed_job(DDTRACE_UNUSED void *_unused) {
     return; // We're not on the main Ractor; we currently don't support profiling non-main Ractors
   }
 
+  state->during_sample = true;
+
   // Rescue against any exceptions that happen during sampling
   safely_call(rescued_sample_from_postponed_job, state->self_instance, state->self_instance);
+
+  state->during_sample = false;
 }
 
 static VALUE rescued_sample_from_postponed_job(VALUE self_instance) {
@@ -672,8 +696,12 @@ static void after_gc_from_postponed_job(DDTRACE_UNUSED void *_unused) {
     return; // We're not on the main Ractor; we currently don't support profiling non-main Ractors
   }
 
+  state->during_sample = true;
+
   // Trigger sampling using the Collectors::ThreadState; rescue against any exceptions that happen during sampling
   safely_call(thread_context_collector_sample_after_gc, state->thread_context_collector_instance, state->self_instance);
+
+  state->during_sample = false;
 }
 
 // Equivalent to Ruby begin/rescue call, where we call a C function and jump to the exception handler if an
@@ -753,6 +781,7 @@ static VALUE _native_stats(DDTRACE_UNUSED VALUE self, VALUE instance) {
     ID2SYM(rb_intern("sampling_time_ns_max")),                       /* => */ pretty_sampling_time_ns_max,
     ID2SYM(rb_intern("sampling_time_ns_total")),                     /* => */ pretty_sampling_time_ns_total,
     ID2SYM(rb_intern("sampling_time_ns_avg")),                       /* => */ pretty_sampling_time_ns_avg,
+    ID2SYM(rb_intern("allocations_during_sample")),                  /* => */ UINT2NUM(state->stats.allocations_during_sample),
   };
   for (long unsigned int i = 0; i < VALUE_COUNT(arguments); i += 2) rb_hash_aset(stats_as_hash, arguments[i], arguments[i+1]);
   return stats_as_hash;
@@ -813,9 +842,45 @@ static void on_newobj_event(DDTRACE_UNUSED VALUE tracepoint_data, DDTRACE_UNUSED
   } else {
     allocation_count++;
   }
+
+  struct cpu_and_wall_time_worker_state *state = active_sampler_instance_state; // Read from global variable, see "sampler global state safety" note above
+
+  // This should not happen in a normal situation because the tracepoint is always enabled after the instance is set
+  // and disabled before it is cleared, but just in case...
+  if (state == NULL) return;
+
+  // In a few cases, we may actually be allocating an object as part of profiler sampling. We don't want to recursively
+  // sample, so we just return early
+  if (state->during_sample) {
+    state->stats.allocations_during_sample++;
+    return;
+  }
+
+  // @ivoanjo: Strictly speaking, this is not needed because Ruby should not call the same tracepoint while a previous
+  // invocation is still pending, (e.g. it wouldn't call `on_newobj_event` while it's already running), but I decided
+  // to keep this here for consistency -- every call to the thread context (other than the special gc calls which are
+  // defined as not being able to allocate) sets this.
+  state->during_sample = true;
+
+  // TODO: Sampling goes here (calling into `thread_context_collector_sample_allocation`)
+
+  state->during_sample = false;
 }
 
 static void disable_tracepoints(struct cpu_and_wall_time_worker_state *state) {
   rb_tracepoint_disable(state->gc_tracepoint);
   rb_tracepoint_disable(state->object_allocation_tracepoint);
 }
+
+static VALUE _native_with_blocked_sigprof(DDTRACE_UNUSED VALUE self) {
+  block_sigprof_signal_handler_from_running_in_current_thread();
+  int exception_state;
+  VALUE result = rb_protect(rb_yield, Qundef, &exception_state);
+  unblock_sigprof_signal_handler_from_running_in_current_thread();
+
+  if (exception_state) {
+    rb_jump_tag(exception_state);
+  } else {
+    return result;
+  }
+}

data/ext/ddtrace_profiling_native_extension/collectors_thread_context.c

@@ -97,6 +97,8 @@ struct thread_context_collector_state {
   unsigned int sample_count;
   // Reusable array to get list of threads
   VALUE thread_list_buffer;
+  // Used to omit endpoint names (retrieved from tracer) from collected data
+  bool endpoint_collection_enabled;
 
   struct stats {
     // Track how many garbage collection samples we've taken.
@@ -140,7 +142,14 @@ static void thread_context_collector_typed_data_free(void *state_ptr);
 static int hash_map_per_thread_context_mark(st_data_t key_thread, st_data_t _value, st_data_t _argument);
 static int hash_map_per_thread_context_free_values(st_data_t _thread, st_data_t value_per_thread_context, st_data_t _argument);
 static VALUE _native_new(VALUE klass);
-static VALUE _native_initialize(VALUE self, VALUE collector_instance, VALUE recorder_instance, VALUE max_frames, VALUE tracer_context_key);
+static VALUE _native_initialize(
+  VALUE self,
+  VALUE collector_instance,
+  VALUE recorder_instance,
+  VALUE max_frames,
+  VALUE tracer_context_key,
+  VALUE endpoint_collection_enabled
+);
 static VALUE _native_sample(VALUE self, VALUE collector_instance, VALUE profiler_overhead_stack_thread);
 static VALUE _native_on_gc_start(VALUE self, VALUE collector_instance);
 static VALUE _native_on_gc_finish(VALUE self, VALUE collector_instance);
@@ -198,7 +207,7 @@ void collectors_thread_context_init(VALUE profiling_module) {
   // https://bugs.ruby-lang.org/issues/18007 for a discussion around this.
   rb_define_alloc_func(collectors_thread_context_class, _native_new);
 
-  rb_define_singleton_method(collectors_thread_context_class, "_native_initialize", _native_initialize, 4);
+  rb_define_singleton_method(collectors_thread_context_class, "_native_initialize", _native_initialize, 5);
   rb_define_singleton_method(collectors_thread_context_class, "_native_inspect", _native_inspect, 1);
   rb_define_singleton_method(collectors_thread_context_class, "_native_reset_after_fork", _native_reset_after_fork, 1);
   rb_define_singleton_method(testing_module, "_native_sample", _native_sample, 2);
@@ -284,11 +293,21 @@ static VALUE _native_new(VALUE klass) {
   state->recorder_instance = Qnil;
   state->tracer_context_key = MISSING_TRACER_CONTEXT_KEY;
   state->thread_list_buffer = rb_ary_new();
+  state->endpoint_collection_enabled = true;
 
   return TypedData_Wrap_Struct(klass, &thread_context_collector_typed_data, state);
 }
 
-static VALUE _native_initialize(DDTRACE_UNUSED VALUE _self, VALUE collector_instance, VALUE recorder_instance, VALUE max_frames, VALUE tracer_context_key) {
+static VALUE _native_initialize(
+  DDTRACE_UNUSED VALUE _self,
+  VALUE collector_instance,
+  VALUE recorder_instance,
+  VALUE max_frames,
+  VALUE tracer_context_key,
+  VALUE endpoint_collection_enabled
+) {
+  ENFORCE_BOOLEAN(endpoint_collection_enabled);
+
   struct thread_context_collector_state *state;
   TypedData_Get_Struct(collector_instance, struct thread_context_collector_state, &thread_context_collector_typed_data, state);
 
@@ -299,6 +318,7 @@ static VALUE _native_initialize(DDTRACE_UNUSED VALUE _self, VALUE collector_inst
   state->sampling_buffer = sampling_buffer_new(max_frames_requested);
   // hash_map_per_thread_context is already initialized, nothing to do here
   state->recorder_instance = enforce_recorder_instance(recorder_instance);
+  state->endpoint_collection_enabled = (endpoint_collection_enabled == Qtrue);
 
   if (RTEST(tracer_context_key)) {
     ENFORCE_TYPE(tracer_context_key, T_SYMBOL);
@@ -732,6 +752,7 @@ static VALUE _native_inspect(DDTRACE_UNUSED VALUE _self, VALUE collector_instanc
   rb_str_concat(result, rb_sprintf(" tracer_context_key=%+"PRIsVALUE, tracer_context_key));
   rb_str_concat(result, rb_sprintf(" sample_count=%u", state->sample_count));
   rb_str_concat(result, rb_sprintf(" stats=%"PRIsVALUE, stats_as_ruby_hash(state)));
+  rb_str_concat(result, rb_sprintf(" endpoint_collection_enabled=%"PRIsVALUE, state->endpoint_collection_enabled ? Qtrue : Qfalse));
 
   return result;
 }
@@ -911,6 +932,8 @@ static void trace_identifiers_for(struct thread_context_collector_state *state,
 
   trace_identifiers_result->valid = true;
 
+  if (!state->endpoint_collection_enabled) return;
+
   VALUE root_span_type = rb_ivar_get(root_span, at_type_id /* @type */);
   if (root_span_type == Qnil || !is_type_web(root_span_type)) return;
 

data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c

@@ -19,6 +19,8 @@
   #pragma GCC diagnostic push
   #pragma GCC diagnostic ignored "-Wunused-parameter"
   #pragma GCC diagnostic ignored "-Wattributes"
+  #pragma GCC diagnostic ignored "-Wpragmas"
+  #pragma GCC diagnostic ignored "-Wexpansion-to-defined"
     #include <vm_core.h>
   #pragma GCC diagnostic pop
   #include <iseq.h>
@@ -651,6 +653,7 @@ check_method_entry(VALUE obj, int can_be_svar)
         if (can_be_svar) {
             return check_method_entry(((struct vm_svar *)obj)->cref_or_me, FALSE);
         }
+        // fallthrough
       default:
 #if VM_CHECK_MODE > 0
         rb_bug("check_method_entry: svar should not be there:");

data/lib/datadog/appsec/component.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative 'processor'
+require_relative 'processor/rule_merger'
+require_relative 'processor/rule_loader'
 
 module Datadog
   module AppSec
@@ -10,14 +12,27 @@ module Datadog
         def build_appsec_component(settings)
           return unless settings.respond_to?(:appsec) && settings.appsec.enabled
 
-          processor = create_processor
+          processor = create_processor(settings)
           new(processor: processor)
         end
 
         private
 
-        def create_processor
-          processor = Processor.new
+        def create_processor(settings)
+          rules = AppSec::Processor::RuleLoader.load_rules(ruleset: settings.appsec.ruleset)
+          return nil unless rules
+
+          data = AppSec::Processor::RuleLoader.load_data(
+            ip_denylist: settings.appsec.ip_denylist,
+            user_id_denylist: settings.appsec.user_id_denylist
+          )
+
+          ruleset = AppSec::Processor::RuleMerger.merge(
+            rules: [rules],
+            data: data,
+          )
+
+          processor = Processor.new(ruleset: ruleset)
           return nil unless processor.ready?
 
           processor
@@ -28,12 +43,31 @@ module Datadog
 
       def initialize(processor:)
         @processor = processor
+        @mutex = Mutex.new
+      end
+
+      def reconfigure(ruleset:)
+        @mutex.synchronize do
+          new = Processor.new(ruleset: ruleset)
+
+          if new && new.ready?
+            old = @processor
+            @processor = new
+            old.finalize
+          end
+        end
+      end
+
+      def reconfigure_lock(&block)
+        @mutex.synchronize(&block)
       end
 
       def shutdown!
-        if processor && processor.ready?
-          processor.finalize
-          @processor = nil
+        @mutex.synchronize do
+          if processor && processor.ready?
+            processor.finalize
+            @processor = nil
+          end
         end
       end
     end

data/lib/datadog/appsec/configuration/settings.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
+require 'set'
+
 module Datadog
   module AppSec
     module Configuration
@@ -87,8 +91,8 @@ module Datadog
         end
 
         # rubocop:disable Layout/LineLength
-        DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'.freeze
-        DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'.freeze
+        DEFAULT_OBFUSCATOR_KEY_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization'
+        DEFAULT_OBFUSCATOR_VALUE_REGEX = '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|token|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\s*=[^;]|"\s*:\s*"[^"]+")|bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\w=-]+\.ey[I-L][\w=-]+(?:\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}'
         # rubocop:enable Layout/LineLength
 
         DEFAULTS = {
@@ -116,9 +120,14 @@ module Datadog
 
         def initialize
           @integrations = []
+          # Stores which options have been configured using Datadog.configure block or ENV variables
+          @configured = Set.new
           @options = DEFAULTS.dup.tap do |options|
             ENVS.each do |env, (key, conv)|
-              options[key] = conv.call(ENV[env]) if ENV[env]
+              if ENV[env]
+                options[key] = conv.call(ENV[env])
+                @configured << key
+              end
             end
           end
         end
@@ -182,7 +191,10 @@ module Datadog
 
         def merge(dsl)
           dsl.options.each do |k, v|
-            @options[k] = v unless v.nil?
+            unless v.nil?
+              @options[k] = v
+              @configured << k
+            end
           end
 
           return self unless @options[:enabled]
@@ -206,6 +218,10 @@ module Datadog
 
         private
 
+        def default?(option)
+          !@configured.include?(option)
+        end
+
         # Restore to original state, for testing only.
         def reset!
           initialize

data/lib/datadog/appsec/configuration.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'configuration/settings'
 
 module Datadog
@@ -82,6 +84,12 @@ module Datadog
         def settings
           @settings ||= Settings.new
         end
+
+        private
+
+        def default_setting?(setting)
+          settings.send(:default?, setting)
+        end
       end
     end
   end

data/lib/datadog/appsec/contrib/auto_instrument.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/configuration/settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../../../core/configuration/base'
 
 module Datadog

data/lib/datadog/appsec/contrib/integration.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib

data/lib/datadog/appsec/contrib/patcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Datadog
   module AppSec
     module Contrib