datadog 2.40.0 → 2.42.0

data/lib/datadog/ai_guard/evaluation/request.rb

@@ -36,10 +36,10 @@ module Datadog
                 messages: @serialized_messages,
                 meta: {
                   service: Datadog.configuration.service,
-                  env: Datadog.configuration.env
-                }
-              }
-            }
+                  env: Datadog.configuration.env,
+                },
+              },
+            },
           }
         end
 
@@ -56,10 +56,10 @@ module Datadog
                   id: message.tool_call.id,
                   function: {
                     name: message.tool_call.tool_name,
-                    arguments: message.tool_call.arguments
-                  }
-                }
-              ]
+                    arguments: message.tool_call.arguments,
+                  },
+                },
+              ],
             }
           elsif message.content.is_a?(::Array)
             {role: message.role, content: serialize_content_parts(message.content)}

data/lib/datadog/ai_guard/evaluation.rb

@@ -51,7 +51,7 @@ module Datadog
                 messages: truncate_content(truncate_messages(request.serialized_messages)),
                 attack_categories: result.tags,
                 sds: result.sds_findings,
-                tag_probs: result.tag_probabilities
+                tag_probs: result.tag_probabilities,
               }
             )
 
@@ -88,7 +88,7 @@ module Datadog
             if message[:content].is_a?(::Array)
               serialized_content = message[:content].map do |part|
                 if part[:text]
-                  {**part, text: part[:text].to_s.byteslice(0, max_bytes)}
+                  {**part, text: truncate_content_value(part[:text].to_s, max_bytes)}
                 else
                   part
                 end
@@ -98,11 +98,18 @@ module Datadog
             else
               {
                 **message,
-                content: message[:content].byteslice(0, max_bytes)
+                content: truncate_content_value(message[:content], max_bytes),
               }
             end
           end
         end
+
+        def truncate_content_value(content, max_bytes)
+          truncated = content.byteslice(0, max_bytes).to_s
+          return truncated if truncated.valid_encoding?
+
+          truncated.scrub("")
+        end
       end
     end
   end

data/lib/datadog/ai_guard/ext.rb

@@ -24,7 +24,7 @@ module Datadog
       TRACE_ANOMALY_DETECTION_TAGS = [
         TRACE_HTTP_USERAGENT_TAG,
         TRACE_HTTP_CLIENT_IP_TAG,
-        TRACE_NETWORK_CLIENT_IP_TAG
+        TRACE_NETWORK_CLIENT_IP_TAG,
       ].freeze
     end
   end

data/lib/datadog/appsec/actions_handler/serializable_backtrace.rb

@@ -3,7 +3,7 @@
 module Datadog
   module AppSec
     module ActionsHandler
-      # This module serves encapsulates MessagePack serialization for caller locations.
+      # Encapsulates serialisation of caller locations.
       #
       # It serializes part of the stack:
       # up to 32 frames (configurable)
@@ -21,50 +21,21 @@ module Datadog
         def to_msgpack(packer = nil)
           # JRuby doesn't pass the packer
           packer ||= MessagePack::Packer.new
-
-          packer.write_map_header(3)
-
-          packer.write("id")
-          packer.write(@stack_id.encode("UTF-8"))
-
-          packer.write("language")
-          packer.write("ruby".encode("UTF-8"))
-
-          serializable_locations_map = build_serializable_locations_map
-
-          packer.write("frames")
-          packer.write_array_header(serializable_locations_map.size)
-
-          serializable_locations_map.each do |frame_id, location|
-            packer.write_map_header(6)
-
-            packer.write("id")
-            packer.write(frame_id)
-
-            packer.write("text")
-            packer.write(location.to_s.encode("UTF-8"))
-
-            packer.write("file")
-            packer.write(location.path&.encode("UTF-8"))
-
-            packer.write("line")
-            packer.write(location.lineno)
-
-            class_name, function_name = location.label&.match(CLASS_AND_FUNCTION_NAME_REGEX)&.captures
-
-            packer.write("class_name")
-            packer.write(class_name&.encode("UTF-8"))
-
-            packer.write("function")
-            packer.write(function_name&.encode("UTF-8"))
-          end
-
+          packer.write(to_h)
           packer
         end
 
+        def to_h
+          {
+            "id" => @stack_id.encode("UTF-8"),
+            "language" => "ruby".encode("UTF-8"),
+            "frames" => build_serializable_frames,
+          }
+        end
+
         private
 
-        def build_serializable_locations_map
+        def build_serializable_frames
           max_depth = Datadog.configuration.appsec.stack_trace.max_depth
           top_percent = Datadog.configuration.appsec.stack_trace.top_percentage
 
@@ -72,7 +43,7 @@ module Datadog
           drop_until_idx = @locations.size - (max_depth - drop_from_idx)
 
           frame_idx = -1
-          @locations.each_with_object({}) do |location, map|
+          @locations.each_with_object([]) do |location, frames|
             # we are dropping frames from library code without increasing frame index
             next if location.path&.include?("lib/datadog")
 
@@ -80,7 +51,15 @@ module Datadog
 
             next if max_depth != 0 && frame_idx >= drop_from_idx && frame_idx < drop_until_idx
 
-            map[frame_idx] = location
+            class_name, function_name = location.label&.match(CLASS_AND_FUNCTION_NAME_REGEX)&.captures
+            frames << {
+              "id" => frame_idx,
+              "text" => location.to_s.encode("UTF-8"),
+              "file" => location.path&.encode("UTF-8"),
+              "line" => location.lineno,
+              "class_name" => class_name&.encode("UTF-8"),
+              "function" => function_name&.encode("UTF-8"),
+            }
           end
         end
       end

data/lib/datadog/appsec/api_security/endpoint_collection/grape_route_serializer.rb

@@ -16,7 +16,7 @@ module Datadog
               resource_name: "#{route.request_method} #{path}",
               operation_name: "http.request",
               method: route.request_method,
-              path: path
+              path: path,
             }
           end
         end

data/lib/datadog/appsec/api_security/endpoint_collection/rails_route_serializer.rb

@@ -26,7 +26,7 @@ module Datadog
               resource_name: "#{method} #{path}",
               operation_name: "http.request",
               method: method,
-              path: path
+              path: path,
             }
           end
         end

data/lib/datadog/appsec/api_security/endpoint_collection/sinatra_route_serializer.rb

@@ -16,7 +16,7 @@ module Datadog
               resource_name: "#{method} #{path}",
               operation_name: "http.request",
               method: method,
-              path: path
+              path: path,
             }
           end
         end

data/lib/datadog/appsec/assets/blocked.html

@@ -1,108 +1 @@
-<!-- Sorry, you've been blocked -->
-<!DOCTYPE html>
-<html lang="en">
-
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width,initial-scale=1">
-    <title>You've been blocked</title>
-    <style>
-        a,
-        body,
-        div,
-        html,
-        span {
-            margin: 0;
-            padding: 0;
-            border: 0;
-            font-size: 100%;
-            font: inherit;
-            vertical-align: baseline
-        }
-
-        body {
-            background: -webkit-radial-gradient(26% 19%, circle, #fff, #f4f7f9);
-            background: radial-gradient(circle at 26% 19%, #fff, #f4f7f9);
-            display: -webkit-box;
-            display: -ms-flexbox;
-            display: flex;
-            -webkit-box-pack: center;
-            -ms-flex-pack: center;
-            justify-content: center;
-            -webkit-box-align: center;
-            -ms-flex-align: center;
-            align-items: center;
-            -ms-flex-line-pack: center;
-            align-content: center;
-            width: 100%;
-            min-height: 100vh;
-            line-height: 1;
-            flex-direction: column
-        }
-
-        p {
-            display: block
-        }
-
-
-        main {
-            text-align: center;
-            flex: 1;
-            display: -webkit-box;
-            display: -ms-flexbox;
-            display: flex;
-            -webkit-box-pack: center;
-            -ms-flex-pack: center;
-            justify-content: center;
-            -webkit-box-align: center;
-            -ms-flex-align: center;
-            align-items: center;
-            -ms-flex-line-pack: center;
-            align-content: center;
-            flex-direction: column
-        }
-
-        p {
-            font-size: 18px;
-            line-height: normal;
-            color: #646464;
-            font-family: sans-serif;
-            font-weight: 400
-        }
-
-        a {
-            color: #4842b7
-        }
-
-        footer {
-            width: 100%;
-            text-align: center
-        }
-
-        footer p {
-            font-size: 16px
-        }
-
-        .security-response-id {
-            font-size:14px;
-            color:#999;
-            margin-top:20px;
-            font-family:monospace
-        }
-    </style>
-</head>
-
-<body>
-    <main>
-        <p>Sorry, you cannot access this page. Please contact the customer service team.</p>
-        <p class="security-response-id">Security Response ID: [security_response_id]</p>
-    </main>
-    <footer>
-        <p>Security provided by <a
-                href="https://www.datadoghq.com/product/security-platform/application-security-monitoring/"
-                target="_blank"
-                rel="noopener noreferrer">Datadog</a></p>
-    </footer>
-</body>
-
-</html>
+<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>You've been blocked</title><style>a,body,div,html,span{margin:0;padding:0;border:0;font-size:100%;font:inherit;vertical-align:baseline}body{background:-webkit-radial-gradient(26% 19%,circle,#fff,#f4f7f9);background:radial-gradient(circle at 26% 19%,#fff,#f4f7f9);display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-ms-flex-line-pack:center;align-content:center;width:100%;min-height:100vh;line-height:1;flex-direction:column}p{display:block}main{text-align:center;flex:1;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-ms-flex-line-pack:center;align-content:center;flex-direction:column}p{font-size:18px;line-height:normal;color:#646464;font-family:sans-serif;font-weight:400}a{color:#4842b7}footer{width:100%;text-align:center}footer p{font-size:16px}.security-response-id{font-size:14px;color:#999;margin-top:20px;font-family:monospace}</style></head><body><main><p>Sorry, you cannot access this page. Please contact the customer service team.</p><p class="security-response-id">Security Response ID: [security_response_id]</p></main><footer><p>Security provided by <a href="https://www.datadoghq.com/product/security-platform/application-security-monitoring/" target="_blank" rel="noopener noreferrer">Datadog</a></p></footer></body></html>

data/lib/datadog/appsec/configuration.rb

@@ -20,7 +20,7 @@ module Datadog
         AUTO_USER_INSTRUMENTATION_MODES = [
           DISABLED_AUTO_USER_INSTRUMENTATION_MODE,
           ANONYMIZATION_AUTO_USER_INSTRUMENTATION_MODE,
-          IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE
+          IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE,
         ].freeze
         AUTO_USER_INSTRUMENTATION_MODES_ALIASES = {
           "ident" => IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE,
@@ -31,7 +31,7 @@ module Datadog
         SAFE_TRACK_USER_EVENTS_MODE = "safe"
         EXTENDED_TRACK_USER_EVENTS_MODE = "extended"
         APPSEC_VALID_TRACK_USER_EVENTS_MODE = [
-          SAFE_TRACK_USER_EVENTS_MODE, EXTENDED_TRACK_USER_EVENTS_MODE
+          SAFE_TRACK_USER_EVENTS_MODE, EXTENDED_TRACK_USER_EVENTS_MODE,
         ].freeze
         APPSEC_VALID_TRACK_USER_EVENTS_ENABLED_VALUES = ["1", "true"].concat(
           APPSEC_VALID_TRACK_USER_EVENTS_MODE

data/lib/datadog/appsec/context.rb

@@ -62,7 +62,7 @@ module Datadog
         @state = {
           events: [],
           interrupted: false,
-          downstream_body_analyzed_count: 0
+          downstream_body_analyzed_count: 0,
         }
       end
 

data/lib/datadog/appsec/contrib/active_record/instrumentation.rb

@@ -25,7 +25,7 @@ module Datadog
 
             ephemeral_data = {
               "server.db.statement" => sql,
-              "server.db.system" => db_system
+              "server.db.system" => db_system,
             }
 
             waf_timeout = Datadog.configuration.appsec.waf_timeout

data/lib/datadog/appsec/contrib/aws_lambda/waf_addresses.rb

@@ -35,7 +35,7 @@ module Datadog
               "server.request.body" => parse_body(payload, headers),
               "server.request.body.byte_length" => body_byte_length(payload),
               "server.request.path_params" => payload["path_params"],
-              "http.client_ip" => extract_client_ip(payload["source_ip"], headers)
+              "http.client_ip" => extract_client_ip(payload["source_ip"], headers),
             }
 
             data.compact!
@@ -51,7 +51,7 @@ module Datadog
               "server.response.headers" => headers,
               "server.response.headers.no_cookies" => headers.dup.tap { |h| h.delete("set-cookie") },
               "server.response.body" => parse_body(payload, headers),
-              "server.response.body.byte_length" => body_byte_length(payload)
+              "server.response.body.byte_length" => body_byte_length(payload),
             }
 
             data.compact!

data/lib/datadog/appsec/contrib/devise/configuration.rb

@@ -11,7 +11,7 @@ module Datadog
             AppSec::Configuration::Settings::SAFE_TRACK_USER_EVENTS_MODE =>
               AppSec::Configuration::Settings::ANONYMIZATION_AUTO_USER_INSTRUMENTATION_MODE,
             AppSec::Configuration::Settings::EXTENDED_TRACK_USER_EVENTS_MODE =>
-              AppSec::Configuration::Settings::IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE
+              AppSec::Configuration::Settings::IDENTIFICATION_AUTO_USER_INSTRUMENTATION_MODE,
           }.freeze
 
           module_function

data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb

@@ -29,7 +29,7 @@ module Datadog
             ephemeral_data = {
               "server.io.net.url" => url,
               "server.io.net.request.method" => data[:method].to_s.upcase,
-              "server.io.net.request.headers" => headers
+              "server.io.net.request.headers" => headers,
             }
 
             is_redirect = context.state[:downstream_redirect_url] == url
@@ -60,7 +60,7 @@ module Datadog
             # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
               "server.io.net.response.status" => data.dig(:response, :status).to_s,
-              "server.io.net.response.headers" => headers
+              "server.io.net.response.headers" => headers,
             }
 
             is_redirect = REDIRECT_STATUS_CODES.cover?(data.dig(:response, :status)) && headers.key?("location")

data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb

@@ -26,7 +26,7 @@ module Datadog
             ephemeral_data = {
               "server.io.net.url" => url,
               "server.io.net.request.method" => env.method.to_s.upcase,
-              "server.io.net.request.headers" => headers
+              "server.io.net.request.headers" => headers,
             }
 
             is_redirect = context.state[:downstream_redirect_url] == url
@@ -56,7 +56,7 @@ module Datadog
             # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
               "server.io.net.response.status" => env.status.to_s,
-              "server.io.net.response.headers" => headers
+              "server.io.net.response.headers" => headers,
             }
 
             is_redirect = REDIRECT_STATUS_CODES.cover?(env.status) && headers.key?("location")

data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb

@@ -27,7 +27,7 @@ module Datadog
 
                   if context
                     persistent_data = {
-                      "graphql.server.all_resolvers" => gateway_multiplex.arguments
+                      "graphql.server.all_resolvers" => gateway_multiplex.arguments,
                     }
 
                     result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)

data/lib/datadog/appsec/contrib/rack/ext.rb

@@ -17,7 +17,7 @@ module Datadog
             "x-appgw-trace-id",
             "x-cloud-trace-context",
             "x-sigsci-requestid",
-            "x-sigsci-tags"
+            "x-sigsci-tags",
           ].freeze
 
           IDENTITY_COLLECTABLE_REQUEST_HEADERS = [
@@ -38,7 +38,7 @@ module Datadog
             "x-cluster-client-ip",
             "x-forwarded",
             "x-forwarded-for",
-            "x-real-ip"
+            "x-real-ip",
           ].freeze
         end
       end

data/lib/datadog/appsec/contrib/rack/gateway/watcher.rb

@@ -34,7 +34,7 @@ module Datadog
                     "server.request.headers" => gateway_request.headers,
                     "server.request.headers.no_cookies" => gateway_request.headers.dup.tap { |h| h.delete("cookie") },
                     "http.client_ip" => gateway_request.client_ip,
-                    "server.request.method" => gateway_request.method
+                    "server.request.method" => gateway_request.method,
                   }
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
@@ -63,7 +63,7 @@ module Datadog
                   persistent_data = {
                     "server.response.status" => gateway_response.status.to_s,
                     "server.response.headers" => gateway_response.headers,
-                    "server.response.headers.no_cookies" => gateway_response.headers.dup.tap { |h| h.delete("set-cookie") }
+                    "server.response.headers.no_cookies" => gateway_response.headers.dup.tap { |h| h.delete("set-cookie") },
                   }
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)

data/lib/datadog/appsec/contrib/rails/gateway/watcher.rb

@@ -26,7 +26,7 @@ module Datadog
                   limit = Datadog.configuration.appsec.body_parsing_size_limit
 
                   persistent_data = {
-                    "server.request.path_params" => gateway_request.route_params
+                    "server.request.path_params" => gateway_request.route_params,
                   }
 
                   unless limit.zero?
@@ -67,7 +67,7 @@ module Datadog
                   context = container.context
 
                   persistent_data = {
-                    "server.response.body" => container.data
+                    "server.response.body" => container.data,
                   }
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 

data/lib/datadog/appsec/contrib/rest_client/request_ssrf_detection_patch.rb

@@ -24,7 +24,7 @@ module Datadog
             ephemeral_data = {
               "server.io.net.url" => url,
               "server.io.net.request.method" => method.to_s.upcase,
-              "server.io.net.request.headers" => headers
+              "server.io.net.request.headers" => headers,
             }
 
             is_redirect = context.state[:downstream_redirect_url] == url
@@ -68,7 +68,7 @@ module Datadog
             # @type var ephemeral_data: ::Datadog::AppSec::Context::input_data
             ephemeral_data = {
               "server.io.net.response.status" => response.code.to_s,
-              "server.io.net.response.headers" => headers
+              "server.io.net.response.headers" => headers,
             }
 
             is_redirect = REDIRECT_STATUS_CODES.cover?(response.code.to_i) && headers.key?("location")

data/lib/datadog/appsec/contrib/sinatra/gateway/watcher.rb

@@ -77,7 +77,7 @@ module Datadog
                   context = gateway_request.env[AppSec::Ext::CONTEXT_KEY] # : Context
 
                   persistent_data = {
-                    "server.request.path_params" => gateway_route_params.params
+                    "server.request.path_params" => gateway_route_params.params,
                   }
 
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
@@ -102,7 +102,7 @@ module Datadog
                   context = container.context # : Context
 
                   persistent_data = {
-                    "server.response.body" => container.data
+                    "server.response.body" => container.data,
                   }
                   result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 

data/lib/datadog/appsec/processor/rule_loader.rb

@@ -59,7 +59,7 @@ module Datadog
             {
               "id" => id,
               "type" => "data_with_expiration",
-              "data" => denylist.map { |v| {"value" => v.to_s, "expiration" => 2**63} }
+              "data" => denylist.map { |v| {"value" => v.to_s, "expiration" => 2**63} },
             }
           end
 
@@ -82,12 +82,12 @@ module Datadog
                   "parameters" => {
                     "inputs" => [
                       {
-                        "address" => "http.client_ip"
-                      }
+                        "address" => "http.client_ip",
+                      },
                     ],
-                    "list" => pass
-                  }
-                }
+                    "list" => pass,
+                  },
+                },
               ],
               "id" => SecureRandom.uuid,
             }
@@ -99,15 +99,15 @@ module Datadog
                   "parameters" => {
                     "inputs" => [
                       {
-                        "address" => "http.client_ip"
-                      }
+                        "address" => "http.client_ip",
+                      },
                     ],
-                    "list" => monitor
-                  }
-                }
+                    "list" => monitor,
+                  },
+                },
               ],
               "id" => SecureRandom.uuid,
-              "on_match" => "monitor"
+              "on_match" => "monitor",
             }
 
             exclusions

data/lib/datadog/appsec/response.rb

@@ -43,7 +43,7 @@ module Datadog
               content(
                 security_response_id: interrupt_params["security_response_id"],
                 content_type: content_type
-              )
+              ),
             ],
           )
         end

data/lib/datadog/appsec/route_normalizer/route_pattern.rb

@@ -31,7 +31,7 @@ module Datadog
         OPTIONAL_GROUP_SIGILS = [
           GROUP_OPEN_CHAR,
           GROUP_CLOSE_CHAR,
-          OPTIONAL_GROUP_SUFFIX_CHAR
+          OPTIONAL_GROUP_SUFFIX_CHAR,
         ].join
 
         NAMED_PARAM_PREFIX_CHAR = ":"
@@ -41,14 +41,14 @@ module Datadog
           NAMED_PARAM_PREFIX_CHAR,
           GLOB_PARAM_PREFIX_CHAR,
           GROUP_OPEN_CHAR,
-          GROUP_CLOSE_CHAR
+          GROUP_CLOSE_CHAR,
         ].join
 
         PARAM_NAME_CHARS = [*"a".."z", *"A".."Z", *"0".."9", "_"].join
 
         EXCLUDED_PARAM_NAME_TERMINATOR_CHARS = [
           PARAM_NAME_CHARS,
-          PATTERN_STRUCTURE_CHARS
+          PATTERN_STRUCTURE_CHARS,
         ].join
 
         MAX_RESOLVE_LENGTH = 8192

data/lib/datadog/appsec/security_engine/engine.rb

@@ -31,7 +31,7 @@ module Datadog
           @waf_builder = WAF::HandleBuilder.new(
             obfuscator: {
               key_regex: appsec_settings.obfuscator_key_regex,
-              value_regex: appsec_settings.obfuscator_value_regex
+              value_regex: appsec_settings.obfuscator_value_regex,
             }
           )
 
@@ -130,7 +130,7 @@ module Datadog
             tags: {
               waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
               event_rules_version: ruleset_version.to_s,
-              success: success.to_s
+              success: success.to_s,
             }
           )
         end
@@ -156,7 +156,7 @@ module Datadog
           common_tags = {
             waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
             event_rules_version: diagnostics["ruleset_version"].to_s,
-            action: action
+            action: action,
           }
 
           if diagnostics["error"]

data/lib/datadog/core/configuration/components.rb

@@ -248,6 +248,7 @@ module Datadog
           crashtracker&.update_on_fork
           ProcessDiscovery.after_fork
           symbol_database&.after_fork!
+          data_streams&.restart_flush_thread
         end
 
         # Hot-swaps with a new sampler.
@@ -352,13 +353,13 @@ module Datadog
           # have direct ownership of it.
           old_statsd = [
             runtime_metrics.metrics.statsd,
-            health_metrics.statsd
+            health_metrics.statsd,
           ].compact.uniq
 
           new_statsd = if replacement
             [
               replacement.runtime_metrics.metrics.statsd,
-              replacement.health_metrics.statsd
+              replacement.health_metrics.statsd,
             ].compact.uniq
           else
             []