ddtrace 1.8.0 → 1.9.0

data/ext/ddtrace_profiling_native_extension/native_extension_helpers.rb

@@ -2,7 +2,7 @@
 
 # typed: ignore
 
-require 'libdatadog'
+require 'rubygems'
 require 'pathname'
 
 module Datadog
@@ -17,6 +17,8 @@ module Datadog
       # Older Rubies don't have the MJIT header, used by the JIT compiler, so we need to use a different approach
       CAN_USE_MJIT_HEADER = RUBY_VERSION >= '2.6'
 
+      LIBDATADOG_VERSION = '~> 1.0.1.1.0'
+
       def self.fail_install_if_missing_extension?
         ENV[ENV_FAIL_INSTALL_IF_MISSING_EXTENSION].to_s.strip.downcase == 'true'
       end
@@ -88,6 +90,7 @@ module Datadog
             not_on_amd64_or_arm64? ||
             on_ruby_2_1? ||
             expected_to_use_mjit_but_mjit_is_disabled? ||
+            libdatadog_not_available? ||
             libdatadog_not_usable?
         end
 
@@ -142,11 +145,6 @@ module Datadog
           '<https://dtdg.co/ruby-profiler-troubleshooting>.'
         ].freeze
 
-        REPORT_ISSUE = [
-          'If you needed to use this, please tell us why on',
-          '<https://github.com/DataDog/dd-trace-rb/issues/new> so we can fix it :)',
-        ].freeze
-
         GET_IN_TOUCH = [
           "Get in touch with us if you're interested in profiling your app!"
         ].freeze
@@ -172,10 +170,10 @@ module Datadog
         PKG_CONFIG_IS_MISSING = explain_issue(
           #+-----------------------------------------------------------------------------+
           'the `pkg-config` system tool is missing.',
-          'This issue can usually be fixed by installing:',
-          '1. the `pkg-config` package on Homebrew and Debian/Ubuntu-based Linux;',
-          '2. the `pkgconf` package on Arch and Alpine-based Linux;',
-          '3. the `pkgconf-pkg-config` package on Fedora/Red Hat-based Linux.',
+          'This issue can usually be fixed by installing one of the following:',
+          'the `pkg-config` package on Homebrew and Debian/Ubuntu-based Linux;',
+          'the `pkgconf` package on Arch and Alpine-based Linux;',
+          'the `pkgconf-pkg-config` package on Fedora/Red Hat-based Linux.',
           suggested: CONTACT_SUPPORT,
         )
 
@@ -188,10 +186,15 @@ module Datadog
         )
 
         private_class_method def self.disabled_via_env?
+          report_disabled = [
+            'If you needed to use this, please tell us why on',
+            '<https://github.com/DataDog/dd-trace-rb/issues/new> so we can fix it :)',
+          ].freeze
+
           disabled_via_env = explain_issue(
             'the `DD_PROFILING_NO_EXTENSION` environment variable is/was set to',
             '`true` during installation.',
-            suggested: REPORT_ISSUE,
+            suggested: report_disabled,
           )
 
           return unless ENV[ENV_NO_EXTENSION].to_s.strip.downcase == 'true'
@@ -254,7 +257,7 @@ module Datadog
             suggested: GET_IN_TOUCH,
           )
 
-          architecture_not_supported unless RUBY_PLATFORM.start_with?('x86_64', 'aarch64')
+          architecture_not_supported unless RUBY_PLATFORM.start_with?('x86_64', 'aarch64', 'arm64')
         end
 
         private_class_method def self.on_ruby_2_1?
@@ -279,6 +282,25 @@ module Datadog
           ruby_without_mjit if CAN_USE_MJIT_HEADER && RbConfig::CONFIG['MJIT_SUPPORT'] != 'yes'
         end
 
+        private_class_method def self.libdatadog_not_available?
+          begin
+            gem 'libdatadog', LIBDATADOG_VERSION
+            require 'libdatadog'
+            nil
+          # rubocop:disable Lint/RescueException
+          rescue Exception => e
+            explain_issue(
+              'there was an exception during loading of the `libdatadog` gem:',
+              e.class.name,
+              *e.message.split("\n"),
+              *Array(e.backtrace),
+              '.',
+              suggested: CONTACT_SUPPORT,
+            )
+          end
+          # rubocop:enable Lint/RescueException
+        end
+
         private_class_method def self.libdatadog_not_usable?
           no_binaries_for_current_platform = explain_issue(
             'the `libdatadog` gem installed on your system is missing binaries for your',

data/ext/ddtrace_profiling_native_extension/private_vm_api_access.c

@@ -390,6 +390,7 @@ calc_lineno(const rb_iseq_t *iseq, const VALUE *pc)
 //   for iseqs created from calls to `eval` and `instance_eval`. This makes it so that `rb_profile_frame_path` on
 //   the `VALUE` returned by rb_profile_frames returns `(eval)` instead of the path of the file where the `eval`
 //   was called from.
+// * Imported fix from https://github.com/ruby/ruby/pull/7116 to avoid sampling threads that are still being created
 //
 // **IMPORTANT: WHEN CHANGING THIS FUNCTION, CONSIDER IF THE SAME CHANGE ALSO NEEDS TO BE MADE TO THE VARIANT FOR
 // RUBY 2.2 AND BELOW WHICH IS ALSO PRESENT ON THIS FILE**
@@ -436,6 +437,10 @@ int ddtrace_rb_profile_frames(VALUE thread, int start, int limit, VALUE *buff, i
     const rb_control_frame_t *cfp = ec->cfp, *end_cfp = RUBY_VM_END_CONTROL_FRAME(ec);
     const rb_callable_method_entry_t *cme;
 
+    // This should not happen for ddtrace (it can only happen when a thread is still being created), but I've imported
+    // it from https://github.com/ruby/ruby/pull/7116 in a "just in case" kind of mindset.
+    if (cfp == NULL) return 0;
+
     // Avoid sampling dead threads
     if (th->status == THREAD_KILLED) return 0;
 
@@ -747,6 +752,7 @@ calc_lineno(const rb_iseq_t *iseq, const VALUE *pc)
 //   `vm_backtrace.c` (`backtrace_each`, `backtrace_size`, `rb_ec_partial_backtrace_object`) but are conspicuously
 //   absent from `rb_profile_frames`. Oversight?
 // * Check thread status and do not sample if thread has been killed.
+// * Imported fix from https://github.com/ruby/ruby/pull/7116 to avoid sampling threads that are still being created
 //
 // The `rb_profile_frames` function changed quite a bit between Ruby 2.2 and 2.3. Since the change was quite complex
 // I opted not to try to extend support to Ruby 2.2 using the same custom function, and instead I started
@@ -760,6 +766,10 @@ int ddtrace_rb_profile_frames(VALUE thread, int start, int limit, VALUE *buff, i
     rb_thread_t *th = thread_struct_from_object(thread);
     rb_control_frame_t *cfp = th->cfp, *end_cfp = RUBY_VM_END_CONTROL_FRAME(th);
 
+    // This should not happen for ddtrace (it can only happen when a thread is still being created), but I've imported
+    // it from https://github.com/ruby/ruby/pull/7116 in a "just in case" kind of mindset.
+    if (cfp == NULL) return 0;
+
     // Avoid sampling dead threads
     if (th->status == THREAD_KILLED) return 0;
 

data/ext/ddtrace_profiling_native_extension/stack_recorder.c

@@ -7,7 +7,7 @@
 #include "libdatadog_helpers.h"
 #include "ruby_helpers.h"
 
-// Used to wrap a ddog_Profile in a Ruby object and expose Ruby-level serialization APIs
+// Used to wrap a ddog_prof_Profile in a Ruby object and expose Ruby-level serialization APIs
 // This file implements the native bits of the Datadog::Profiling::StackRecorder class
 
 // ---
@@ -24,7 +24,7 @@
 // 2. The thread that serializes and reports profiles, let's call it the **serializer thread**. We enforce that there
 // cannot be more than one thread attempting to serialize profiles at a time.
 //
-// If both the sampler and serializer threads are trying to access the same `ddog_Profile` in parallel, we will
+// If both the sampler and serializer threads are trying to access the same `ddog_prof_Profile` in parallel, we will
 // have a concurrency issue. Thus, the StackRecorder has an added mechanism to avoid this.
 //
 // As an additional constraint, the **sampler thread** has absolute priority and must never block while
@@ -32,7 +32,7 @@
 //
 // ### The solution: Keep two profiles at the same time
 //
-// To solve for the constraints above, the StackRecorder keeps two `ddog_Profile` profile instances inside itself.
+// To solve for the constraints above, the StackRecorder keeps two `ddog_prof_Profile` profile instances inside itself.
 // They are called the `slot_one_profile` and `slot_two_profile`.
 //
 // Each profile is paired with its own mutex. `slot_one_profile` is protected by `slot_one_mutex` and `slot_two_profile`
@@ -135,10 +135,10 @@ static VALUE stack_recorder_class = Qnil;
 // Contains native state for each instance
 struct stack_recorder_state {
   pthread_mutex_t slot_one_mutex;
-  ddog_Profile *slot_one_profile;
+  ddog_prof_Profile *slot_one_profile;
 
   pthread_mutex_t slot_two_mutex;
-  ddog_Profile *slot_two_profile;
+  ddog_prof_Profile *slot_two_profile;
 
   short active_slot; // MUST NEVER BE ACCESSED FROM record_sample; this is NOT for the sampler thread to use.
 };
@@ -146,7 +146,7 @@ struct stack_recorder_state {
 // Used to return a pair of values from sampler_lock_active_profile()
 struct active_slot_pair {
   pthread_mutex_t *mutex;
-  ddog_Profile *profile;
+  ddog_prof_Profile *profile;
 };
 
 struct call_serialize_without_gvl_arguments {
@@ -155,8 +155,8 @@ struct call_serialize_without_gvl_arguments {
   ddog_Timespec finish_timestamp;
 
   // Set by callee
-  ddog_Profile *profile;
-  ddog_SerializeResult result;
+  ddog_prof_Profile *profile;
+  ddog_prof_Profile_SerializeResult result;
 
   // Set by both
   bool serialize_ran;
@@ -170,7 +170,7 @@ static VALUE ruby_time_from(ddog_Timespec ddprof_time);
 static void *call_serialize_without_gvl(void *call_args);
 static struct active_slot_pair sampler_lock_active_profile();
 static void sampler_unlock_active_profile(struct active_slot_pair active_slot);
-static ddog_Profile *serializer_flip_active_and_inactive_slots(struct stack_recorder_state *state);
+static ddog_prof_Profile *serializer_flip_active_and_inactive_slots(struct stack_recorder_state *state);
 static VALUE _native_active_slot(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance);
 static VALUE _native_is_slot_one_mutex_locked(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance);
 static VALUE _native_is_slot_two_mutex_locked(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance);
@@ -207,7 +207,7 @@ void stack_recorder_init(VALUE profiling_module) {
   ruby_time_from_id = rb_intern_const("ruby_time_from");
 }
 
-// This structure is used to define a Ruby object that stores a pointer to a ddog_Profile instance
+// This structure is used to define a Ruby object that stores a pointer to a ddog_prof_Profile instance
 // See also https://github.com/ruby/ruby/blob/master/doc/extension.rdoc for how this works
 static const rb_data_type_t stack_recorder_typed_data = {
   .wrap_struct_name = "Datadog::Profiling::StackRecorder",
@@ -222,14 +222,14 @@ static const rb_data_type_t stack_recorder_typed_data = {
 static VALUE _native_new(VALUE klass) {
   struct stack_recorder_state *state = ruby_xcalloc(1, sizeof(struct stack_recorder_state));
 
-  ddog_Slice_value_type sample_types = {.ptr = enabled_value_types, .len = ENABLED_VALUE_TYPES_COUNT};
+  ddog_prof_Slice_ValueType sample_types = {.ptr = enabled_value_types, .len = ENABLED_VALUE_TYPES_COUNT};
 
   initialize_slot_concurrency_control(state);
 
   // Note: Don't raise exceptions after this point, since it'll lead to libdatadog memory leaking!
 
-  state->slot_one_profile = ddog_Profile_new(sample_types, NULL /* period is optional */, NULL /* start_time is optional */);
-  state->slot_two_profile = ddog_Profile_new(sample_types, NULL /* period is optional */, NULL /* start_time is optional */);
+  state->slot_one_profile = ddog_prof_Profile_new(sample_types, NULL /* period is optional */, NULL /* start_time is optional */);
+  state->slot_two_profile = ddog_prof_Profile_new(sample_types, NULL /* period is optional */, NULL /* start_time is optional */);
 
   return TypedData_Wrap_Struct(klass, &stack_recorder_typed_data, state);
 }
@@ -248,10 +248,10 @@ static void stack_recorder_typed_data_free(void *state_ptr) {
   struct stack_recorder_state *state = (struct stack_recorder_state *) state_ptr;
 
   pthread_mutex_destroy(&state->slot_one_mutex);
-  ddog_Profile_free(state->slot_one_profile);
+  ddog_prof_Profile_drop(state->slot_one_profile);
 
   pthread_mutex_destroy(&state->slot_two_mutex);
-  ddog_Profile_free(state->slot_two_profile);
+  ddog_prof_Profile_drop(state->slot_two_profile);
 
   ruby_xfree(state);
 }
@@ -282,26 +282,26 @@ static VALUE _native_serialize(DDTRACE_UNUSED VALUE _self, VALUE recorder_instan
     rb_thread_call_without_gvl2(call_serialize_without_gvl, &args, NULL /* No interruption function needed in this case */, NULL /* Not needed */);
   }
 
-  ddog_SerializeResult serialized_profile = args.result;
+  ddog_prof_Profile_SerializeResult serialized_profile = args.result;
 
-  if (serialized_profile.tag == DDOG_SERIALIZE_RESULT_ERR) {
-    VALUE err_details = ruby_string_from_vec_u8(serialized_profile.err);
-    ddog_SerializeResult_drop(serialized_profile);
+  if (serialized_profile.tag == DDOG_PROF_PROFILE_SERIALIZE_RESULT_ERR) {
+    VALUE err_details = ruby_string_from_prof_vec_u8(serialized_profile.err);
+    ddog_prof_Profile_SerializeResult_drop(serialized_profile);
     return rb_ary_new_from_args(2, error_symbol, err_details);
   }
 
-  VALUE encoded_pprof = ruby_string_from_vec_u8(serialized_profile.ok.buffer);
+  VALUE encoded_pprof = ruby_string_from_prof_vec_u8(serialized_profile.ok.buffer);
 
   ddog_Timespec ddprof_start = serialized_profile.ok.start;
   ddog_Timespec ddprof_finish = serialized_profile.ok.end;
 
   // Clean up libdatadog object to avoid leaking in case ruby_time_from raises an exception
-  ddog_SerializeResult_drop(serialized_profile);
+  ddog_prof_Profile_SerializeResult_drop(serialized_profile);
 
   VALUE start = ruby_time_from(ddprof_start);
   VALUE finish = ruby_time_from(ddprof_finish);
 
-  if (!ddog_Profile_reset(args.profile, NULL /* start_time is optional */ )) {
+  if (!ddog_prof_Profile_reset(args.profile, NULL /* start_time is optional */ )) {
     return rb_ary_new_from_args(2, error_symbol, rb_str_new_cstr("Failed to reset profile"));
   }
 
@@ -318,13 +318,13 @@ static VALUE ruby_time_from(ddog_Timespec ddprof_time) {
   #endif
 }
 
-void record_sample(VALUE recorder_instance, ddog_Sample sample) {
+void record_sample(VALUE recorder_instance, ddog_prof_Sample sample) {
   struct stack_recorder_state *state;
   TypedData_Get_Struct(recorder_instance, struct stack_recorder_state, &stack_recorder_typed_data, state);
 
   struct active_slot_pair active_slot = sampler_lock_active_profile(state);
 
-  ddog_Profile_add(active_slot.profile, sample);
+  ddog_prof_Profile_add(active_slot.profile, sample);
 
   sampler_unlock_active_profile(active_slot);
 }
@@ -335,7 +335,7 @@ void record_endpoint(VALUE recorder_instance, ddog_CharSlice local_root_span_id,
 
   struct active_slot_pair active_slot = sampler_lock_active_profile(state);
 
-  ddog_Profile_set_endpoint(active_slot.profile, local_root_span_id, endpoint);
+  ddog_prof_Profile_set_endpoint(active_slot.profile, local_root_span_id, endpoint);
 
   sampler_unlock_active_profile(active_slot);
 }
@@ -344,7 +344,7 @@ static void *call_serialize_without_gvl(void *call_args) {
   struct call_serialize_without_gvl_arguments *args = (struct call_serialize_without_gvl_arguments *) call_args;
 
   args->profile = serializer_flip_active_and_inactive_slots(args->state);
-  args->result = ddog_Profile_serialize(args->profile, &args->finish_timestamp, NULL /* duration_nanos is optional */);
+  args->result = ddog_prof_Profile_serialize(args->profile, &args->finish_timestamp, NULL /* duration_nanos is optional */);
   args->serialize_ran = true;
 
   return NULL; // Unused
@@ -382,7 +382,7 @@ static void sampler_unlock_active_profile(struct active_slot_pair active_slot) {
   ENFORCE_SUCCESS_GVL(pthread_mutex_unlock(active_slot.mutex));
 }
 
-static ddog_Profile *serializer_flip_active_and_inactive_slots(struct stack_recorder_state *state) {
+static ddog_prof_Profile *serializer_flip_active_and_inactive_slots(struct stack_recorder_state *state) {
   int previously_active_slot = state->active_slot;
 
   if (previously_active_slot != 1 && previously_active_slot != 2) {
@@ -466,19 +466,19 @@ static VALUE _native_reset_after_fork(DDTRACE_UNUSED VALUE self, VALUE recorder_
   // resulting state is inconsistent, we make sure to reset it back to the initial state.
   initialize_slot_concurrency_control(state);
 
-  ddog_Profile_reset(state->slot_one_profile, /* start_time: */ NULL);
-  ddog_Profile_reset(state->slot_two_profile, /* start_time: */ NULL);
+  ddog_prof_Profile_reset(state->slot_one_profile, /* start_time: */ NULL);
+  ddog_prof_Profile_reset(state->slot_two_profile, /* start_time: */ NULL);
 
   return Qtrue;
 }
 
-// Assumption 1: This method is called with the GVL being held, because `ddog_Profile_reset` mutates the profile and should
+// Assumption 1: This method is called with the GVL being held, because `ddog_prof_Profile_reset` mutates the profile and should
 // not be interrupted part-way through by a VM fork.
 static void serializer_set_start_timestamp_for_next_profile(struct stack_recorder_state *state, ddog_Timespec timestamp) {
   // Before making this profile active, we reset it so that it uses the correct timestamp for its start
-  ddog_Profile *next_profile = (state->active_slot == 1) ? state->slot_two_profile : state->slot_one_profile;
+  ddog_prof_Profile *next_profile = (state->active_slot == 1) ? state->slot_two_profile : state->slot_one_profile;
 
-  if (!ddog_Profile_reset(next_profile, &timestamp)) rb_raise(rb_eRuntimeError, "Failed to reset profile");
+  if (!ddog_prof_Profile_reset(next_profile, &timestamp)) rb_raise(rb_eRuntimeError, "Failed to reset profile");
 }
 
 static VALUE _native_record_endpoint(DDTRACE_UNUSED VALUE _self, VALUE recorder_instance, VALUE local_root_span_id, VALUE endpoint) {

data/ext/ddtrace_profiling_native_extension/stack_recorder.h

@@ -10,7 +10,7 @@
 // ```
 // compiling ../../../../ext/ddtrace_profiling_native_extension/stack_recorder.c
 // ../../../../ext/ddtrace_profiling_native_extension/stack_recorder.c:23:1: error: initializer element is not constant
-// static const ddog_ValueType enabled_value_types[] = {CPU_TIME_VALUE, CPU_SAMPLES_VALUE, WALL_TIME_VALUE};
+// static const ddog_prof_ValueType enabled_value_types[] = {CPU_TIME_VALUE, CPU_SAMPLES_VALUE, WALL_TIME_VALUE};
 // ^
 // ```
 #define VALUE_STRING(string) {.ptr = "" string, .len = sizeof(string) - 1}
@@ -23,7 +23,7 @@
 #define HEAP_LIVE_SIZE_VALUE    {.type_ = VALUE_STRING("heap-live-size"),    .unit = VALUE_STRING("bytes")}
 #define HEAP_LIVE_SAMPLES_VALUE {.type_ = VALUE_STRING("heap-live-samples"), .unit = VALUE_STRING("count")}
 
-static const ddog_ValueType enabled_value_types[] = {
+static const ddog_prof_ValueType enabled_value_types[] = {
   #define CPU_TIME_VALUE_POS 0
   CPU_TIME_VALUE,
   #define CPU_SAMPLES_VALUE_POS 1
@@ -32,8 +32,8 @@ static const ddog_ValueType enabled_value_types[] = {
   WALL_TIME_VALUE
 };
 
-#define ENABLED_VALUE_TYPES_COUNT (sizeof(enabled_value_types) / sizeof(ddog_ValueType))
+#define ENABLED_VALUE_TYPES_COUNT (sizeof(enabled_value_types) / sizeof(ddog_prof_ValueType))
 
-void record_sample(VALUE recorder_instance, ddog_Sample sample);
+void record_sample(VALUE recorder_instance, ddog_prof_Sample sample);
 void record_endpoint(VALUE recorder_instance, ddog_CharSlice local_root_span_id, ddog_CharSlice endpoint);
 VALUE enforce_recorder_instance(VALUE object);

data/lib/datadog/appsec/assets/waf_rules/recommended.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.4.2"
+    "rules_version": "1.4.3"
   },
   "rules": [
     {
@@ -1802,7 +1802,7 @@
                 "address": "server.request.path_params"
               }
             ],
-            "regex": "^(?i:file|ftps?|https?).*?\\?+$",
+            "regex": "^(?i:file|ftps?|http)://.*?\\?+$",
             "options": {
               "case_sensitive": true,
               "min_length": 4
@@ -2694,8 +2694,9 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "\\b(?:s(?:e(?:t(?:_(?:e(?:xception|rror)_handler|magic_quotes_runtime|include_path)|defaultstub)|ssion_s(?:et_save_handler|tart))|qlite_(?:(?:(?:unbuffered|single|array)_)?query|create_(?:aggregate|function)|p?open|exec)|tr(?:eam_(?:context_create|socket_client)|ipc?slashes|rev)|implexml_load_(?:string|file)|ocket_c(?:onnect|reate)|h(?:ow_sourc|a1_fil)e|pl_autoload_register|ystem)|p(?:r(?:eg_(?:replace(?:_callback(?:_array)?)?|match(?:_all)?|split)|oc_(?:(?:terminat|clos|nic)e|get_status|open)|int_r)|o(?:six_(?:get(?:(?:e[gu]|g)id|login|pwnam)|mk(?:fifo|nod)|ttyname|kill)|pen)|hp(?:_(?:strip_whitespac|unam)e|version|info)|g_(?:(?:execut|prepar)e|connect|query)|a(?:rse_(?:ini_file|str)|ssthru)|utenv)|r(?:unkit_(?:function_(?:re(?:defin|nam)e|copy|add)|method_(?:re(?:defin|nam)e|copy|add)|constant_(?:redefine|add))|e(?:(?:gister_(?:shutdown|tick)|name)_function|ad(?:(?:gz)?file|_exif_data|dir))|awurl(?:de|en)code)|i(?:mage(?:createfrom(?:(?:jpe|pn)g|x[bp]m|wbmp|gif)|(?:jpe|pn)g|g(?:d2?|if)|2?wbmp|xbm)|s_(?:(?:(?:execut|write?|read)ab|fi)le|dir)|ni_(?:get(?:_all)?|set)|terator_apply|ptcembed)|g(?:et(?:_(?:c(?:urrent_use|fg_va)r|meta_tags)|my(?:[gpu]id|inode)|(?:lastmo|cw)d|imagesize|env)|z(?:(?:(?:defla|wri)t|encod|fil)e|compress|open|read)|lob)|a(?:rray_(?:u(?:intersect(?:_u?assoc)?|diff(?:_u?assoc)?)|intersect_u(?:assoc|key)|diff_u(?:assoc|key)|filter|reduce|map)|ssert(?:_options)?|lert|tob)|h(?:tml(?:specialchars(?:_decode)?|_entity_decode|entities)|(?:ash(?:_(?:update|hmac))?|ighlight)_file|e(?:ader_register_callback|x2bin))|f(?:i(?:le(?:(?:[acm]tim|inod)e|(?:_exist|perm)s|group)?|nfo_open)|tp_(?:nb_(?:ge|pu)|connec|ge|pu)t|(?:unction_exis|pu)ts|write|open)|o(?:b_(?:get_(?:c(?:ontents|lean)|flush)|end_(?:clean|flush)|clean|flush|start)|dbc_(?:result(?:_all)?|exec(?:ute)?|connect)|pendir)|m(?:b_(?:ereg(?:_(?:replace(?:_callback)?|match)|i(?:_replace)?)?|parse_str)|(?:ove_uploaded|d5)_file|ethod_exists|ysql_query|kdir)|e(?:x(?:if_(?:t(?:humbnail|agname)|imagetype|read_data)|ec)|scapeshell(?:arg|cmd)|rror_reporting|val)|c(?:url_(?:file_create|exec|init)|onvert_uuencode|reate_function|hr)|u(?:n(?:serialize|pack)|rl(?:de|en)code|[ak]?sort)|b(?:(?:son_(?:de|en)|ase64_en)code|zopen|toa)|(?:json_(?:de|en)cod|debug_backtrac|tmpfil)e|var_dump)(?:\\s|/\\*.*\\*/|//.*|#.*|\\\")*\\(.*\\)",
+            "regex": "\\b(?:s(?:e(?:t(?:_(?:e(?:xception|rror)_handler|magic_quotes_runtime|include_path)|defaultstub)|ssion_s(?:et_save_handler|tart))|qlite_(?:(?:(?:unbuffered|single|array)_)?query|create_(?:aggregate|function)|p?open|exec)|tr(?:eam_(?:context_create|socket_client)|ipc?slashes|rev)|implexml_load_(?:string|file)|ocket_c(?:onnect|reate)|h(?:ow_sourc|a1_fil)e|pl_autoload_register|ystem)|p(?:r(?:eg_(?:replace(?:_callback(?:_array)?)?|match(?:_all)?|split)|oc_(?:(?:terminat|clos|nic)e|get_status|open)|int_r)|o(?:six_(?:get(?:(?:e[gu]|g)id|login|pwnam)|mk(?:fifo|nod)|ttyname|kill)|pen)|hp(?:_(?:strip_whitespac|unam)e|version|info)|g_(?:(?:execut|prepar)e|connect|query)|a(?:rse_(?:ini_file|str)|ssthru)|utenv)|r(?:unkit_(?:function_(?:re(?:defin|nam)e|copy|add)|method_(?:re(?:defin|nam)e|copy|add)|constant_(?:redefine|add))|e(?:(?:gister_(?:shutdown|tick)|name)_function|ad(?:(?:gz)?file|_exif_data|dir))|awurl(?:de|en)code)|i(?:mage(?:createfrom(?:(?:jpe|pn)g|x[bp]m|wbmp|gif)|(?:jpe|pn)g|g(?:d2?|if)|2?wbmp|xbm)|s_(?:(?:(?:execut|write?|read)ab|fi)le|dir)|ni_(?:get(?:_all)?|set)|terator_apply|ptcembed)|g(?:et(?:_(?:c(?:urrent_use|fg_va)r|meta_tags)|my(?:[gpu]id|inode)|(?:lastmo|cw)d|imagesize|env)|z(?:(?:(?:defla|wri)t|encod|fil)e|compress|open|read)|lob)|a(?:rray_(?:u(?:intersect(?:_u?assoc)?|diff(?:_u?assoc)?)|intersect_u(?:assoc|key)|diff_u(?:assoc|key)|filter|reduce|map)|ssert(?:_options)?|lert|tob)|h(?:tml(?:specialchars(?:_decode)?|_entity_decode|entities)|(?:ash(?:_(?:update|hmac))?|ighlight)_file|e(?:ader_register_callback|x2bin))|f(?:i(?:le(?:(?:[acm]tim|inod)e|(?:_exist|perm)s|group)?|nfo_open)|tp_(?:nb_(?:ge|pu)|connec|ge|pu)t|(?:unction_exis|pu)ts|write|open)|o(?:b_(?:get_(?:c(?:ontents|lean)|flush)|end_(?:clean|flush)|clean|flush|start)|dbc_(?:result(?:_all)?|exec(?:ute)?|connect)|pendir)|m(?:b_(?:ereg(?:_(?:replace(?:_callback)?|match)|i(?:_replace)?)?|parse_str)|(?:ove_uploaded|d5)_file|ethod_exists|ysql_query|kdir)|e(?:x(?:if_(?:t(?:humbnail|agname)|imagetype|read_data)|ec)|scapeshell(?:arg|cmd)|rror_reporting|val)|c(?:url_(?:file_create|exec|init)|onvert_uuencode|reate_function|hr)|u(?:n(?:serialize|pack)|rl(?:de|en)code|[ak]?sort)|b(?:(?:son_(?:de|en)|ase64_en)code|zopen|toa)|(?:json_(?:de|en)cod|debug_backtrac|tmpfil)e|var_dump)(?:\\s|/\\*.*\\*/|//.*|#.*|\\\"|')*\\((?:(?:\\s|/\\*.*\\*/|//.*|#.*)*(?:\\$\\w+|[A-Z\\d]\\w*|\\w+\\(.*\\)|\\\\?\"(?:[^\"]|\\\\\"|\"\"|\"\\+\")*\\\\?\"|\\\\?'(?:[^']|''|'\\+')*\\\\?')(?:\\s|/\\*.*\\*/|//.*|#.*)*(?:(?:::|\\.|->)(?:\\s|/\\*.*\\*/|//.*|#.*)*\\w+(?:\\(.*\\))?)?,)*(?:(?:\\s|/\\*.*\\*/|//.*|#.*)*(?:\\$\\w+|[A-Z\\d]\\w*|\\w+\\(.*\\)|\\\\?\"(?:[^\"]|\\\\\"|\"\"|\"\\+\")*\\\\?\"|\\\\?'(?:[^']|''|'\\+')*\\\\?')(?:\\s|/\\*.*\\*/|//.*|#.*)*(?:(?:::|\\.|->)(?:\\s|/\\*.*\\*/|//.*|#.*)*\\w+(?:\\(.*\\))?)?)?\\)",
             "options": {
+              "case_sensitive": true,
               "min_length": 5
             }
           },
@@ -3524,7 +3525,7 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "\\b(?i:eval|settimeout|setinterval|new\\s+Function)\\s*\\(",
+            "regex": "\\b(?i:eval|settimeout|setinterval|new\\s+Function|alert|prompt)\\s*\\([^\\)]",
             "options": {
               "case_sensitive": true,
               "min_length": 5
@@ -3770,7 +3771,7 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "(?i:(?:\\[?\\$(?:(?:s(?:lic|iz)|wher)e|e(?:lemMatch|xists|q)|n(?:o[rt]|in?|e)|l(?:ike|te?)|t(?:ext|ype)|a(?:ll|nd)|jsonSchema|between|regex|x?or|div|mod)\\]?))",
+            "regex": "(?i:(?:\\[?\\$(?:(?:s(?:lic|iz)|wher)e|e(?:lemMatch|xists|q)|n(?:o[rt]|in?|e)|l(?:ike|te?)|t(?:ext|ype)|a(?:ll|nd)|jsonSchema|between|regex|x?or|div|mod)\\]?)\\b)",
             "options": {
               "case_sensitive": true,
               "min_length": 3
@@ -3808,7 +3809,7 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "(?:^[\\W\\d]+\\s*?(?:alter\\s*(?:a(?:(?:pplication\\s*rol|ggregat)e|s(?:ymmetric\\s*ke|sembl)y|u(?:thorization|dit)|vailability\\s*group)|c(?:r(?:yptographic\\s*provider|edential)|o(?:l(?:latio|um)|nversio)n|ertificate|luster)|s(?:e(?:rv(?:ice|er)|curity|quence|ssion|arch)|y(?:mmetric\\s*key|nonym)|togroup|chema)|m(?:a(?:s(?:ter\\s*key|k)|terialized)|e(?:ssage\\s*type|thod)|odule)|l(?:o(?:g(?:file\\s*group|in)|ckdown)|a(?:ngua|r)ge|ibrary)|t(?:(?:abl(?:espac)?|yp)e|r(?:igger|usted)|hreshold|ext)|p(?:a(?:rtition|ckage)|ro(?:cedur|fil)e|ermission)|d(?:i(?:mension|skgroup)|atabase|efault|omain)|r(?:o(?:l(?:lback|e)|ute)|e(?:sourc|mot)e)|f(?:u(?:lltext|nction)|lashback|oreign)|e(?:xte(?:nsion|rnal)|(?:ndpoi|ve)nt)|in(?:dex(?:type)?|memory|stance)|b(?:roker\\s*priority|ufferpool)|x(?:ml\\s*schema|srobject)|w(?:ork(?:load)?|rapper)|hi(?:erarchy|stogram)|o(?:perator|utline)|(?:nicknam|queu)e|us(?:age|er)|group|java|view)\\b|(?:(?:(?:trunc|cre)at|renam)e|d(?:e(?:lete|sc)|rop)|(?:inser|selec)t|load)\\s+\\w+|u(?:nion\\s*(?:(?:distin|sele)ct|all)\\b|pdate\\s+\\w+))|\\b(?:(?:(?:(?:trunc|cre|upd)at|renam)e|(?:inser|selec)t|de(?:lete|sc)|alter|load)\\s+(?:group_concat|load_file|char)\\b\\s*\\(?|end\\s*?\\);)|[\\\"'`\\w]\\s+as\\b\\s*[\\\"'`\\w]+\\s*\\bfrom|[\\s(?:]load_file\\s*?\\(|[\\\"'`]\\s+regexp\\W)",
+            "regex": "(?:^[\\W\\d]+\\s*?(?:alter\\s*(?:a(?:(?:pplication\\s*rol|ggregat)e|s(?:ymmetric\\s*ke|sembl)y|u(?:thorization|dit)|vailability\\s*group)|c(?:r(?:yptographic\\s*provider|edential)|o(?:l(?:latio|um)|nversio)n|ertificate|luster)|s(?:e(?:rv(?:ice|er)|curity|quence|ssion|arch)|y(?:mmetric\\s*key|nonym)|togroup|chema)|m(?:a(?:s(?:ter\\s*key|k)|terialized)|e(?:ssage\\s*type|thod)|odule)|l(?:o(?:g(?:file\\s*group|in)|ckdown)|a(?:ngua|r)ge|ibrary)|t(?:(?:abl(?:espac)?|yp)e|r(?:igger|usted)|hreshold|ext)|p(?:a(?:rtition|ckage)|ro(?:cedur|fil)e|ermission)|d(?:i(?:mension|skgroup)|atabase|efault|omain)|r(?:o(?:l(?:lback|e)|ute)|e(?:sourc|mot)e)|f(?:u(?:lltext|nction)|lashback|oreign)|e(?:xte(?:nsion|rnal)|(?:ndpoi|ve)nt)|in(?:dex(?:type)?|memory|stance)|b(?:roker\\s*priority|ufferpool)|x(?:ml\\s*schema|srobject)|w(?:ork(?:load)?|rapper)|hi(?:erarchy|stogram)|o(?:perator|utline)|(?:nicknam|queu)e|us(?:age|er)|group|java|view)|union\\s*(?:(?:distin|sele)ct|all))\\b|\\b(?:(?:(?:trunc|cre|upd)at|renam)e|(?:inser|selec)t|de(?:lete|sc)|alter|load)\\s+(?:group_concat|load_file|char)\\b\\s*\\(?|[\\s(]load_file\\s*?\\(|[\\\"'`]\\s+regexp\\W)",
             "options": {
               "min_length": 5
             }
@@ -4177,7 +4178,7 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "[#%$]{[^}]+[^\\w\\s][^}]+}",
+            "regex": "[#%$]{(?:[^}]+[^\\w\\s}\\-_][^}]+|\\d+-\\d+)}",
             "options": {
               "case_sensitive": true
             }
@@ -4352,6 +4353,38 @@
       ],
       "transformers": []
     },
+    {
+      "id": "dog-931-001",
+      "name": "RFI: URL Payload to well known RFI target",
+      "tags": {
+        "type": "rfi",
+        "category": "attack_attempt"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              }
+            ],
+            "regex": "^(?i:file|ftps?|https?).*/rfiinc\\.txt\\?+$",
+            "options": {
+              "case_sensitive": true,
+              "min_length": 17
+            }
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": []
+    },
     {
       "id": "nfd-000-001",
       "name": "Detect common directory discovery scans",
@@ -5160,7 +5193,7 @@
                 "address": "grpc.server.request.message"
               }
             ],
-            "regex": "^(jar:)?(http|https):\\/\\/([0-9oq]{1,5}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|[0-9]{1,10}|localhost)(:[0-9]{1,5})?(\\/.*|)$"
+            "regex": "^(jar:)?(http|https):\\/\\/([0-9oq]{1,5}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|[0-9]{1,10})(:[0-9]{1,5})?(\\/.*|)$"
           },
           "operator": "match_regex"
         }
@@ -6417,6 +6450,40 @@
       ],
       "transformers": []
     },
+    {
+      "id": "ua0-600-56x",
+      "name": "Datadog test scanner - blocking version: user-agent",
+      "tags": {
+        "type": "security_scanner",
+        "category": "attack_attempt"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "user-agent"
+                ]
+              },
+              {
+                "address": "grpc.server.request.metadata",
+                "key_path": [
+                  "dd-canary"
+                ]
+              }
+            ],
+            "regex": "^dd-test-scanner-log-block$"
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": [],
+      "on_match": [
+        "block"
+      ]
+    },
     {
       "id": "ua0-600-5xx",
       "name": "Blind SQL Injection Brute Forcer",

data/lib/datadog/appsec/assets/waf_rules/risky.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.4.2"
+    "rules_version": "1.4.3"
   },
   "rules": [
     {

data/lib/datadog/appsec/assets/waf_rules/strict.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.4.2"
+    "rules_version": "1.4.3"
   },
   "rules": [
     {

data/lib/datadog/appsec/assets.rb

@@ -1,4 +1,4 @@
-# typed: false
+# typed: ignore
 
 require 'pathname'