datadog 2.14.0 → 2.16.0

data/ext/libdatadog_api/init.c

@@ -0,0 +1,15 @@
+#include <ruby.h>
+
+#include "datadog_ruby_common.h"
+#include "crashtracker.h"
+#include "process_discovery.h"
+#include "library_config.h"
+
+void DDTRACE_EXPORT Init_libdatadog_api(void) {
+  VALUE datadog_module = rb_define_module("Datadog");
+  VALUE core_module = rb_define_module_under(datadog_module, "Core");
+
+  crashtracker_init(core_module);
+  process_discovery_init(core_module);
+  library_config_init(core_module);
+}

data/ext/libdatadog_api/library_config.c

@@ -0,0 +1,122 @@
+#include <ruby.h>
+#include <datadog/library-config.h>
+
+#include "library_config.h"
+#include "datadog_ruby_common.h"
+
+static VALUE _native_configurator_new(VALUE klass);
+static VALUE _native_configurator_get(VALUE self);
+
+static VALUE config_vec_class = Qnil;
+
+// ddog_Configurator memory management
+static void configurator_free(void *configurator_ptr) {
+  ddog_Configurator *configurator = (ddog_Configurator *)configurator_ptr;
+
+  ddog_library_configurator_drop(configurator);
+}
+
+static const rb_data_type_t configurator_typed_data = {
+  .wrap_struct_name = "Datadog::Core::Configuration::StableConfig::Configurator",
+  .function = {
+    .dfree = configurator_free,
+    .dsize = NULL,
+  },
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+// ddog_Vec_LibraryConfig memory management
+static void config_vec_free(void *config_vec_ptr) {
+  ddog_Vec_LibraryConfig *config_vec = (ddog_Vec_LibraryConfig *)config_vec_ptr;
+
+  ddog_library_config_drop(*config_vec);
+  ruby_xfree(config_vec_ptr);
+}
+
+static const rb_data_type_t config_vec_typed_data = {
+  .wrap_struct_name = "Datadog::Core::Configuration::StableConfigVec",
+  .function = {
+    .dfree = config_vec_free,
+    .dsize = NULL,
+  },
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+void library_config_init(VALUE core_module) {
+  rb_global_variable(&config_vec_class);
+  VALUE configuration_module = rb_define_module_under(core_module, "Configuration");
+  VALUE stable_config_module = rb_define_module_under(configuration_module, "StableConfig");
+  VALUE configurator_class = rb_define_class_under(stable_config_module, "Configurator", rb_cObject);
+  config_vec_class = rb_define_class_under(configuration_module, "StableConfigVec", rb_cObject);
+
+  rb_define_alloc_func(configurator_class, _native_configurator_new);
+  rb_define_method(configurator_class, "get", _native_configurator_get, 0);
+
+  rb_undef_alloc_func(config_vec_class); // It cannot be created from Ruby code and only serves as an intermediate object for the Ruby GC
+}
+
+// TODO: After libdatadog 17.1 release, delete rb_raise, uncomment code and change `DDTRACE_UNUSED VALUE _klass` by `VALUE klass`
+static VALUE _native_configurator_new(DDTRACE_UNUSED VALUE _klass) {
+  /*
+  ddog_Configurator *configurator = ddog_library_configurator_new(false, DDOG_CHARSLICE_C("ruby"));
+
+  ddog_library_configurator_with_detect_process_info(configurator);
+
+  return TypedData_Wrap_Struct(klass, &configurator_typed_data, configurator);
+  */
+
+  rb_raise(rb_eNotImpError, "TODO: Not in use yet, waiting for libdatadog 17.1");
+}
+
+static VALUE _native_configurator_get(VALUE self) {
+  ddog_Configurator *configurator;
+  TypedData_Get_Struct(self, ddog_Configurator, &configurator_typed_data, configurator);
+
+  ddog_Result_VecLibraryConfig configurator_result = ddog_library_configurator_get(configurator);
+
+  if (configurator_result.tag == DDOG_RESULT_VEC_LIBRARY_CONFIG_ERR_VEC_LIBRARY_CONFIG) {
+    ddog_Error err = configurator_result.err;
+    VALUE message = get_error_details_and_drop(&err);
+    if (is_config_loaded()) {
+      log_warning(message);
+    } else {
+      log_warning_without_config(message);
+    }
+    return rb_hash_new();
+  }
+
+  // Wrapping config_vec into a Ruby object enables the Ruby GC to manage its memory
+  // We need to allocate memory for config_vec because once it is out of scope, it will be freed (at the end of this function)
+  // So we cannot reference it with &config_vec
+  // We are doing this in case one of the ruby API raises an exception before the end of this function,
+  // so the allocated memory will still be freed
+  ddog_Vec_LibraryConfig *config_vec = ruby_xmalloc(sizeof(ddog_Vec_LibraryConfig));
+  *config_vec = configurator_result.ok;
+  VALUE config_vec_rb = TypedData_Wrap_Struct(config_vec_class, &config_vec_typed_data, config_vec);
+
+  VALUE local_config_hash = rb_hash_new();
+  VALUE fleet_config_hash = rb_hash_new();
+  // TODO: Uncomment next block after libdatadog 17.1 release
+  /*
+  for (uintptr_t i = 0; i < config_vec->len; i++) {
+    ddog_LibraryConfig config = config_vec->ptr[i];
+    VALUE selected_hash;
+    if (config.source == DDOG_LIBRARY_CONFIG_SOURCE_LOCAL_STABLE_CONFIG) {
+      selected_hash = local_config_hash;
+    }
+    else {
+      selected_hash = fleet_config_hash;
+    }
+
+    ddog_CStr name = ddog_library_config_name_to_env(config.name);
+    rb_hash_aset(selected_hash, rb_str_new(name.ptr, name.length), rb_str_new(config.value.ptr, config.value.length));
+  }
+  */
+
+  VALUE result = rb_hash_new();
+  rb_hash_aset(result, ID2SYM(rb_intern("local")), local_config_hash);
+  rb_hash_aset(result, ID2SYM(rb_intern("fleet")), fleet_config_hash);
+
+  RB_GC_GUARD(config_vec_rb);
+  return result;
+}

data/ext/libdatadog_api/library_config.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "datadog_ruby_common.h"
+
+void library_config_init(VALUE core_module);
+
+static inline bool is_config_loaded(void) {
+  VALUE datadog_module = rb_const_get(rb_cObject, rb_intern("Datadog"));
+  VALUE is_config_loaded = rb_funcall(datadog_module, rb_intern("configuration?"), 0);
+
+  return is_config_loaded == Qtrue;
+}
+
+static inline VALUE log_warning_without_config(VALUE warning) {
+  VALUE datadog_module = rb_const_get(rb_cObject, rb_intern("Datadog"));
+  VALUE logger = rb_funcall(datadog_module, rb_intern("logger_without_configuration"), 0);
+
+  return rb_funcall(logger, rb_intern("warn"), 1, warning);
+}

data/ext/libdatadog_api/process_discovery.c

@@ -0,0 +1,117 @@
+#include <errno.h>
+#include <stdlib.h>
+#include <ruby.h>
+#include <datadog/common.h>
+
+#include "datadog_ruby_common.h"
+
+static VALUE _native_store_tracer_metadata(int argc, VALUE *argv, DDTRACE_UNUSED VALUE _self);
+static VALUE _native_to_rb_int(DDTRACE_UNUSED VALUE _self, VALUE tracer_memfd);
+static VALUE _native_close_tracer_memfd(DDTRACE_UNUSED VALUE _self, VALUE tracer_memfd, VALUE logger);
+
+static void tracer_memfd_free(void *ptr) {
+  int *fd = (int *)ptr;
+  if (*fd != -1) {
+    close(*fd);
+  }
+  ruby_xfree(ptr);
+}
+
+static const rb_data_type_t tracer_memfd_type = {
+  .wrap_struct_name = "Datadog::Core::ProcessDiscovery::TracerMemfd",
+  .function = {
+    .dfree = tracer_memfd_free,
+    .dsize = NULL,
+  },
+  .flags = RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+void process_discovery_init(VALUE core_module) {
+  VALUE process_discovery_class = rb_define_class_under(core_module, "ProcessDiscovery", rb_cObject);
+  VALUE tracer_memfd_class = rb_define_class_under(process_discovery_class, "TracerMemfd", rb_cObject);
+  rb_undef_alloc_func(tracer_memfd_class); // Class cannot be instantiated from Ruby
+
+  rb_define_singleton_method(process_discovery_class, "_native_store_tracer_metadata", _native_store_tracer_metadata, -1);
+  rb_define_singleton_method(process_discovery_class, "_native_to_rb_int", _native_to_rb_int, 1);
+  rb_define_singleton_method(process_discovery_class, "_native_close_tracer_memfd", _native_close_tracer_memfd, 2);
+}
+
+// TODO: Remove DDTRACE_UNUSED and rename _self to self once we have updated libdatadog to 17.1
+static VALUE _native_store_tracer_metadata(int argc, VALUE *argv, DDTRACE_UNUSED VALUE _self) {
+  VALUE logger;
+  VALUE options;
+  rb_scan_args(argc, argv, "1:", &logger, &options);
+  if (options == Qnil) options = rb_hash_new();
+
+  VALUE schema_version = rb_hash_fetch(options, ID2SYM(rb_intern("schema_version")));
+  VALUE runtime_id = rb_hash_fetch(options, ID2SYM(rb_intern("runtime_id")));
+  VALUE tracer_language = rb_hash_fetch(options, ID2SYM(rb_intern("tracer_language")));
+  VALUE tracer_version = rb_hash_fetch(options, ID2SYM(rb_intern("tracer_version")));
+  VALUE hostname = rb_hash_fetch(options, ID2SYM(rb_intern("hostname")));
+  VALUE service_name = rb_hash_fetch(options, ID2SYM(rb_intern("service_name")));
+  VALUE service_env = rb_hash_fetch(options, ID2SYM(rb_intern("service_env")));
+  VALUE service_version = rb_hash_fetch(options, ID2SYM(rb_intern("service_version")));
+
+  ENFORCE_TYPE(schema_version, T_FIXNUM);
+  ENFORCE_TYPE(runtime_id, T_STRING);
+  ENFORCE_TYPE(tracer_language, T_STRING);
+  ENFORCE_TYPE(tracer_version, T_STRING);
+  ENFORCE_TYPE(hostname, T_STRING);
+  ENFORCE_TYPE(service_name, T_STRING);
+  ENFORCE_TYPE(service_env, T_STRING);
+  ENFORCE_TYPE(service_version, T_STRING);
+
+  /*
+  ddog_Result_TracerMemfdHandle result = ddog_store_tracer_metadata(
+    (uint8_t) NUM2UINT(schema_version),
+    char_slice_from_ruby_string(runtime_id),
+    char_slice_from_ruby_string(tracer_language),
+    char_slice_from_ruby_string(tracer_version),
+    char_slice_from_ruby_string(hostname),
+    char_slice_from_ruby_string(service_name),
+    char_slice_from_ruby_string(service_env),
+    char_slice_from_ruby_string(service_version)
+  );
+
+  if (result.tag == DDOG_RESULT_TRACER_MEMFD_HANDLE_ERR_TRACER_MEMFD_HANDLE) {
+    rb_funcall(logger, rb_intern("debug"), 1, rb_sprintf("Failed to store the tracer configuration in a memory file descriptor: %"PRIsVALUE, get_error_details_and_drop(&result.err)));
+    return Qnil;
+  }
+
+  // &result.ok is a ddog_TracerMemfdHandle, which is a struct only containing int fd, which is a file descriptor
+  // We should just return the fd
+  int *fd = ruby_xmalloc(sizeof(int));
+
+  *fd = result.ok.fd;
+  VALUE tracer_memfd_class = rb_const_get(self, rb_intern("TracerMemfd"));
+  VALUE tracer_memfd = TypedData_Wrap_Struct(tracer_memfd_class, &tracer_memfd_type, fd);
+  return tracer_memfd;
+  */
+
+  rb_raise(rb_eNotImpError, "TODO: Not in use yet, waiting for libdatadog 17.1");
+}
+
+static VALUE _native_to_rb_int(DDTRACE_UNUSED VALUE _self, VALUE tracer_memfd) {
+  int *fd;
+  TypedData_Get_Struct(tracer_memfd, int, &tracer_memfd_type, fd);
+  return INT2NUM(*fd);
+}
+
+static VALUE _native_close_tracer_memfd(DDTRACE_UNUSED VALUE _self, VALUE tracer_memfd, VALUE logger) {
+  int *fd;
+  TypedData_Get_Struct(tracer_memfd, int, &tracer_memfd_type, fd);
+  if (*fd == -1) {
+    rb_funcall(logger, rb_intern("debug"), 1, rb_sprintf("The tracer configuration memory file descriptor has already been closed"));
+    return Qnil;
+  }
+
+  int close_result = close(*fd);
+  *fd = -1;
+
+  if (close_result == -1) {
+    rb_funcall(logger, rb_intern("debug"), 1, rb_sprintf("Failed to close the tracer configuration memory file descriptor: %s", strerror(errno)));
+    return Qnil;
+  }
+
+  return Qnil;
+}

data/ext/libdatadog_api/process_discovery.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#include "datadog_ruby_common.h"
+
+void process_discovery_init(VALUE core_module);

data/lib/datadog/appsec/actions_handler.rb

@@ -33,7 +33,7 @@ module Datadog
         event_category = Ext::EXPLOIT_PREVENTION_EVENT_CATEGORY
         tag_key = Ext::TAG_METASTRUCT_STACK_TRACE
 
-        existing_stack_data = active_span.get_metastruct_tag(tag_key).dup || { event_category => [] }
+        existing_stack_data = active_span.get_metastruct_tag(tag_key).dup || {event_category => []}
         max_stack_traces = Datadog.configuration.appsec.stack_trace.max_stack_traces
         return if max_stack_traces != 0 && existing_stack_data[event_category].count >= max_stack_traces
 
@@ -42,7 +42,8 @@ module Datadog
         active_span.set_metastruct_tag(tag_key, existing_stack_data)
       end
 
-      def generate_schema(_action_params); end
+      def generate_schema(_action_params)
+      end
     end
   end
 end

data/lib/datadog/appsec/assets/waf_rules/README.md

@@ -1,7 +1,52 @@
-Vendored WAF rules originate from https://github.com/datadog/appsec-event-rules
+AppSec WAF rules based on [appsec-event-rules](https://github.com/datadog/appsec-event-rules) builds
 
-One should check rule compatibility with libddwaf, which is the end consumer of
-these rules.
+## How to update
 
-There might be rules that look to be irrelevant to Ruby as they may still help
-identify bad actors.
+> [!WARNING]
+> This process is a temporary workaround to maintain compatibility with the existing code structure and will be changed.
+
+1. Download `recommended.json` and `strict.json` of the desired version from [appsec-event-rules](https://github.com/datadog/appsec-event-rules) (example: [v1.13.3](https://github.com/DataDog/appsec-event-rules/tree/1.13.3/build))
+2. Run the script below inside `waf_rules` folder to extract scanners and processors into separate files
+
+    ```ruby
+    require 'json'
+
+    recommended_rules = JSON.parse(File.read(File.expand_path('recommended.json', __dir__)))
+    strict_rules = JSON.parse(File.read(File.expand_path('strict.json', __dir__)))
+
+    recommended_processors = recommended_rules.delete('processors')
+    strict_processors = strict_rules.delete('processors')
+
+    if recommended_processors.sort_by { |processor| processor['id'] } !=
+        strict_processors.sort_by { |processor| processor['id'] }
+      raise 'Processors are not the same, unable to extract them'
+    end
+
+    puts 'Extracting processors...'
+    File.open(File.expand_path('processors.json', __dir__), 'wb') do |file|
+      file.write(JSON.pretty_generate(recommended_processors))
+    end
+
+    recommended_scanners = recommended_rules.delete('scanners')
+    strict_scanners = strict_rules.delete('scanners')
+
+    if recommended_scanners.sort_by { |processor| processor['id'] } !=
+        strict_scanners.sort_by { |processor| processor['id'] }
+      raise 'Scanners are not the same, unable to extract them'
+    end
+
+    puts 'Extracting scanners...'
+    File.open(File.expand_path('scanners.json', __dir__), 'wb') do |file|
+      file.write(JSON.pretty_generate(recommended_scanners))
+    end
+
+    puts 'Updating rules...'
+
+    File.open(File.expand_path('recommended.json', __dir__), 'wb') do |file|
+      file.write(JSON.pretty_generate(recommended_rules))
+    end
+
+    File.open(File.expand_path('strict.json', __dir__), 'wb') do |file|
+      file.write(JSON.pretty_generate(strict_rules))
+    end
+    ```

data/lib/datadog/appsec/assets/waf_rules/processors.json

@@ -1,6 +1,57 @@
 [
   {
-    "id": "processor-001",
+    "id": "http-endpoint-fingerprint",
+    "generator": "http_endpoint_fingerprint",
+    "conditions": [
+      {
+        "operator": "exists",
+        "parameters": {
+          "inputs": [
+            {
+              "address": "waf.context.event"
+            },
+            {
+              "address": "server.business_logic.users.login.failure"
+            },
+            {
+              "address": "server.business_logic.users.login.success"
+            }
+          ]
+        }
+      }
+    ],
+    "parameters": {
+      "mappings": [
+        {
+          "method": [
+            {
+              "address": "server.request.method"
+            }
+          ],
+          "uri_raw": [
+            {
+              "address": "server.request.uri.raw"
+            }
+          ],
+          "body": [
+            {
+              "address": "server.request.body"
+            }
+          ],
+          "query": [
+            {
+              "address": "server.request.query"
+            }
+          ],
+          "output": "_dd.appsec.fp.http.endpoint"
+        }
+      ]
+    },
+    "evaluate": false,
+    "output": true
+  },
+  {
+    "id": "extract-content",
     "generator": "extract_schema",
     "conditions": [
       {
@@ -32,10 +83,10 @@
         {
           "inputs": [
             {
-              "address": "server.request.headers.no_cookies"
+              "address": "server.request.cookies"
             }
           ],
-          "output": "_dd.appsec.s.req.headers"
+          "output": "_dd.appsec.s.req.cookies"
         },
         {
           "inputs": [
@@ -56,29 +107,89 @@
         {
           "inputs": [
             {
-              "address": "server.request.cookies"
+              "address": "server.response.body"
             }
           ],
-          "output": "_dd.appsec.s.req.cookies"
+          "output": "_dd.appsec.s.res.body"
         },
         {
           "inputs": [
             {
-              "address": "server.response.headers.no_cookies"
+              "address": "graphql.server.all_resolvers"
             }
           ],
-          "output": "_dd.appsec.s.res.headers"
+          "output": "_dd.appsec.s.graphql.all_resolvers"
         },
         {
           "inputs": [
             {
-              "address": "server.response.body"
+              "address": "graphql.server.resolver"
             }
           ],
-          "output": "_dd.appsec.s.res.body"
+          "output": "_dd.appsec.s.graphql.resolver"
+        }
+      ],
+      "scanners": [
+        {
+          "tags": {
+            "category": "payment"
+          }
+        },
+        {
+          "tags": {
+            "category": "pii"
+          }
+        }
+      ]
+    },
+    "evaluate": false,
+    "output": true
+  },
+  {
+    "id": "extract-headers",
+    "generator": "extract_schema",
+    "conditions": [
+      {
+        "operator": "equals",
+        "parameters": {
+          "inputs": [
+            {
+              "address": "waf.context.processor",
+              "key_path": [
+                "extract-schema"
+              ]
+            }
+          ],
+          "type": "boolean",
+          "value": true
+        }
+      }
+    ],
+    "parameters": {
+      "mappings": [
+        {
+          "inputs": [
+            {
+              "address": "server.request.headers.no_cookies"
+            }
+          ],
+          "output": "_dd.appsec.s.req.headers"
+        },
+        {
+          "inputs": [
+            {
+              "address": "server.response.headers.no_cookies"
+            }
+          ],
+          "output": "_dd.appsec.s.res.headers"
         }
       ],
       "scanners": [
+        {
+          "tags": {
+            "category": "credentials"
+          }
+        },
         {
           "tags": {
             "category": "pii"
@@ -88,5 +199,123 @@
     },
     "evaluate": false,
     "output": true
+  },
+  {
+    "id": "http-header-fingerprint",
+    "generator": "http_header_fingerprint",
+    "conditions": [
+      {
+        "operator": "exists",
+        "parameters": {
+          "inputs": [
+            {
+              "address": "waf.context.event"
+            },
+            {
+              "address": "server.business_logic.users.login.failure"
+            },
+            {
+              "address": "server.business_logic.users.login.success"
+            }
+          ]
+        }
+      }
+    ],
+    "parameters": {
+      "mappings": [
+        {
+          "headers": [
+            {
+              "address": "server.request.headers.no_cookies"
+            }
+          ],
+          "output": "_dd.appsec.fp.http.header"
+        }
+      ]
+    },
+    "evaluate": false,
+    "output": true
+  },
+  {
+    "id": "http-network-fingerprint",
+    "generator": "http_network_fingerprint",
+    "conditions": [
+      {
+        "operator": "exists",
+        "parameters": {
+          "inputs": [
+            {
+              "address": "waf.context.event"
+            },
+            {
+              "address": "server.business_logic.users.login.failure"
+            },
+            {
+              "address": "server.business_logic.users.login.success"
+            }
+          ]
+        }
+      }
+    ],
+    "parameters": {
+      "mappings": [
+        {
+          "headers": [
+            {
+              "address": "server.request.headers.no_cookies"
+            }
+          ],
+          "output": "_dd.appsec.fp.http.network"
+        }
+      ]
+    },
+    "evaluate": false,
+    "output": true
+  },
+  {
+    "id": "session-fingerprint",
+    "generator": "session_fingerprint",
+    "conditions": [
+      {
+        "operator": "exists",
+        "parameters": {
+          "inputs": [
+            {
+              "address": "waf.context.event"
+            },
+            {
+              "address": "server.business_logic.users.login.failure"
+            },
+            {
+              "address": "server.business_logic.users.login.success"
+            }
+          ]
+        }
+      }
+    ],
+    "parameters": {
+      "mappings": [
+        {
+          "cookies": [
+            {
+              "address": "server.request.cookies"
+            }
+          ],
+          "session_id": [
+            {
+              "address": "usr.session_id"
+            }
+          ],
+          "user_id": [
+            {
+              "address": "usr.id"
+            }
+          ],
+          "output": "_dd.appsec.fp.session"
+        }
+      ]
+    },
+    "evaluate": false,
+    "output": true
   }
-]
+]