datadog 2.21.0 → 2.22.0

data/lib/datadog/appsec/assets/waf_rules/README.md

@@ -2,51 +2,45 @@ AppSec WAF rules based on [appsec-event-rules](https://github.com/datadog/appsec
 
 ## How to update
 
-> [!WARNING]
-> This process is a temporary workaround to maintain compatibility with the existing code structure and will be changed.
+In order to update rules, download `recommended.json` and `strict.json` of the desired version from [appsec-event-rules](https://github.com/datadog/appsec-event-rules) (example: [v1.13.3](https://github.com/DataDog/appsec-event-rules/tree/1.13.3/build))
 
-1. Download `recommended.json` and `strict.json` of the desired version from [appsec-event-rules](https://github.com/datadog/appsec-event-rules) (example: [v1.13.3](https://github.com/DataDog/appsec-event-rules/tree/1.13.3/build))
-2. Run the script below inside `waf_rules` folder to extract scanners and processors into separate files
+You can store the following code as a `Rakefile` under `lib/datadog/appsec/assets/waf_rules`
 
-    ```ruby
-    require 'json'
+```ruby
+def download(filename)
+  build_path = 'repos/DataDog/appsec-event-rules/contents/build'
 
-    recommended_rules = JSON.parse(File.read(File.expand_path('recommended.json', __dir__)))
-    strict_rules = JSON.parse(File.read(File.expand_path('strict.json', __dir__)))
+  system("gh api #{build_path}/#{filename} --jq '.content' | base64 -d > #{filename}")
+end
 
-    recommended_processors = recommended_rules.delete('processors')
-    strict_processors = strict_rules.delete('processors')
+task default: :update
 
-    if recommended_processors.sort_by { |processor| processor['id'] } !=
-        strict_processors.sort_by { |processor| processor['id'] }
-      raise 'Processors are not the same, unable to extract them'
-    end
+task :verify_dependencies do
+  next if system('which gh 1>/dev/null')
 
-    puts 'Extracting processors...'
-    File.open(File.expand_path('processors.json', __dir__), 'wb') do |file|
-      file.write(JSON.pretty_generate(recommended_processors))
-    end
+  abort <<~MESSAGE
+    \033[0;33mNOTE: To successfully execute that task make sure you have
+          GitHub CLI installed and authenticated https://cli.github.com/\033[0m
+  MESSAGE
+end
 
-    recommended_scanners = recommended_rules.delete('scanners')
-    strict_scanners = strict_rules.delete('scanners')
+desc 'Update recommended.json and strict.json to the latest version'
+task update: :verify_dependencies do
+  download('strict.json')
+  download('recommended.json')
 
-    if recommended_scanners.sort_by { |processor| processor['id'] } !=
-        strict_scanners.sort_by { |processor| processor['id'] }
-      raise 'Scanners are not the same, unable to extract them'
-    end
+  puts "\033[0;32mSuccess!\033[0m"
+end
+```
 
-    puts 'Extracting scanners...'
-    File.open(File.expand_path('scanners.json', __dir__), 'wb') do |file|
-      file.write(JSON.pretty_generate(recommended_scanners))
-    end
+And run the following command
 
-    puts 'Updating rules...'
+> [!IMPORTANT]
+> To run that command you will need to install GitHub CLI tool and authenticate it
+> See: https://cli.github.com/ (or ddtool)
 
-    File.open(File.expand_path('recommended.json', __dir__), 'wb') do |file|
-      file.write(JSON.pretty_generate(recommended_rules))
-    end
 
-    File.open(File.expand_path('strict.json', __dir__), 'wb') do |file|
-      file.write(JSON.pretty_generate(strict_rules))
-    end
-    ```
+```console
+$ bundle exec rake update
+Success!
+```

data/lib/datadog/appsec/assets/waf_rules/recommended.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.14.2"
+    "rules_version": "1.15.1"
   },
   "rules": [
     {
@@ -2985,7 +2985,7 @@
                 "address": "graphql.server.resolver"
               }
             ],
-            "regex": "\\b(?:(?:l(?:(?:utimes|chmod)(?:Sync)?|(?:stat|ink)Sync)|w(?:rite(?:(?:File|v)(?:Sync)?|Sync)|atchFile)|u(?:n(?:watchFile|linkSync)|times(?:Sync)?)|s(?:(?:ymlink|tat)Sync|pawn(?:File|Sync))|ex(?:ec(?:File(?:Sync)?|Sync)|istsSync)|a(?:ppendFile|ccess)(?:Sync)?|(?:Caveat|Inode)s|open(?:dir)?Sync|new\\s+Function|Availability|\\beval)\\s*\\(|m(?:ain(?:Module\\s*(?:\\W*\\s*(?:constructor|require)|\\[)|\\s*(?:\\W*\\s*(?:constructor|require)|\\[))|kd(?:temp(?:Sync)?|irSync)\\s*\\(|odule\\.exports\\s*=)|c(?:(?:(?:h(?:mod|own)|lose)Sync|reate(?:Write|Read)Stream|p(?:Sync)?)\\s*\\(|o(?:nstructor\\s*(?:\\W*\\s*_load|\\[)|pyFile(?:Sync)?\\s*\\())|f(?:(?:(?:s(?:(?:yncS)?|tatS)|datas(?:yncS)?)ync|ch(?:mod|own)(?:Sync)?)\\s*\\(|u(?:nction\\s*\\(\\s*\\)\\s*{|times(?:Sync)?\\s*\\())|r(?:e(?:(?:ad(?:(?:File|link|dir)?Sync|v(?:Sync)?)|nameSync)\\s*\\(|quire\\s*(?:\\W*\\s*main|\\[))|m(?:Sync)?\\s*\\()|process\\s*(?:\\W*\\s*(?:mainModule|binding)|\\[)|t(?:his\\.constructor|runcateSync\\s*\\()|_(?:\\$\\$ND_FUNC\\$\\$_|_js_function)|global\\s*(?:\\W*\\s*process|\\[)|String\\s*\\.\\s*fromCharCode|binding\\s*\\[)",
+            "regex": "\\b(?:(?:l(?:(?:utimes|chmod)(?:Sync)?|(?:stat|ink)Sync)|w(?:rite(?:(?:File|v)(?:Sync)?|Sync)|atchFile)|u(?:n(?:watchFile|linkSync)|times(?:Sync)?)|s(?:(?:ymlink|tat)Sync|pawn(?:File|Sync))|ex(?:ec(?:File(?:Sync)?|Sync)|istsSync)|a(?:ppendFile|ccess)(?:Sync)?|(?:Caveat|Inode)s|open(?:dir)?Sync|new\\s+Function|Availability|\\beval)\\s*\\(|m(?:ain(?:Module\\s*(?:\\W*\\s*(?:constructor|require)|\\[)|\\s*(?:\\W*\\s*(?:constructor|require)|\\[))|kd(?:temp(?:Sync)?|irSync)\\s*\\(|odule\\.exports\\s*=)|c(?:(?:(?:h(?:mod|own)|lose)Sync|reate(?:Write|Read)Stream|p(?:Sync)?)\\s*\\(|o(?:nstructor\\s*(?:\\W*\\s*_load|\\[)|pyFile(?:Sync)?\\s*\\())|f(?:(?:(?:s(?:(?:yncS)?|tatS)|datas(?:yncS)?)ync|ch(?:mod|own)(?:Sync)?)\\s*\\(|u(?:nction\\s*\\(\\s*\\)\\s*{|times(?:Sync)?\\s*\\())|r(?:e(?:(?:ad(?:(?:File|link|dir)?Sync|v(?:Sync)?)|nameSync)\\s*\\(|quire\\s*(?:\\W*\\s*main\\b|\\[))|m(?:Sync)?\\s*\\()|process\\s*(?:\\W*\\s*(?:mainModule|binding)|\\[)|t(?:his\\.constructor|runcateSync\\s*\\()|_(?:\\$\\$ND_FUNC\\$\\$_|_js_function)|global\\s*(?:\\W*\\s*process|\\[)|String\\s*\\.\\s*fromCharCode|binding\\s*\\[)",
             "options": {
               "case_sensitive": true,
               "min_length": 3
@@ -5539,6 +5539,7 @@
         "confidence": "0",
         "module": "waf"
       },
+      "max_version": "1.24.9",
       "conditions": [
         {
           "parameters": {
@@ -5656,6 +5657,52 @@
       ],
       "transformers": []
     },
+    {
+      "id": "dog-932-110",
+      "name": "Python: Subprocess-based command injection",
+      "tags": {
+        "type": "command_injection",
+        "category": "attack_attempt",
+        "confidence": "0",
+        "module": "waf"
+      },
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
+              },
+              {
+                "address": "server.request.headers.no_cookies"
+              },
+              {
+                "address": "grpc.server.request.message"
+              },
+              {
+                "address": "graphql.server.all_resolvers"
+              },
+              {
+                "address": "graphql.server.resolver"
+              }
+            ],
+            "regex": "(?s)\\bsubprocess\\b.*\\b(?:check_output|run|Popen|call|check_call)\\b",
+            "options": {
+              "case_sensitive": true,
+              "min_length": 14
+            }
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": []
+    },
     {
       "id": "dog-934-001",
       "name": "XXE - XML file loads external entity",
@@ -6625,7 +6672,10 @@
               {
                 "address": "graphql.server.resolver"
               }
-            ]
+            ],
+            "options": {
+              "path-inspection": true
+            }
           },
           "operator": "ssrf_detector"
         }
@@ -8870,6 +8920,271 @@
       "transformers": []
     }
   ],
+  "rules_compat": [
+    {
+      "id": "api-001-100",
+      "name": "JWT: No expiry is present",
+      "tags": {
+        "type": "jwt",
+        "category": "api_security",
+        "confidence": "0",
+        "module": "business-logic"
+      },
+      "min_version": "1.25.0",
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.jwt",
+                "key_path": [
+                  "payload",
+                  "exp"
+                ]
+              }
+            ]
+          },
+          "operator": "!exists"
+        }
+      ],
+      "transformers": [],
+      "output": {
+        "event": false,
+        "keep": false,
+        "attributes": {
+          "_dd.appsec.api.jwt.no_expiry": {
+            "value": 1
+          }
+        }
+      }
+    },
+    {
+      "id": "api-001-110",
+      "name": "JWT: Collect algorithm used",
+      "tags": {
+        "type": "jwt",
+        "category": "api_security",
+        "confidence": "0",
+        "module": "business-logic"
+      },
+      "min_version": "1.25.0",
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.jwt",
+                "key_path": [
+                  "header",
+                  "alg"
+                ]
+              }
+            ]
+          },
+          "operator": "exists"
+        }
+      ],
+      "transformers": [],
+      "output": {
+        "event": false,
+        "keep": false,
+        "attributes": {
+          "_dd.appsec.api.jwt_alg": {
+            "address": "server.request.jwt",
+            "key_path": [
+              "header",
+              "alg"
+            ]
+          }
+        }
+      }
+    },
+    {
+      "id": "api-001-120",
+      "name": "JWT: No audience is specified",
+      "tags": {
+        "type": "jwt",
+        "category": "api_security",
+        "confidence": "0",
+        "module": "business-logic"
+      },
+      "min_version": "1.25.0",
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.jwt",
+                "key_path": [
+                  "payload",
+                  "aud"
+                ]
+              }
+            ]
+          },
+          "operator": "!exists"
+        }
+      ],
+      "transformers": [],
+      "output": {
+        "event": false,
+        "keep": false,
+        "attributes": {
+          "_dd.appsec.api.jwt.no_audience": {
+            "value": 1
+          }
+        }
+      }
+    },
+    {
+      "id": "api-001-130",
+      "name": "JWT: None algorithm used",
+      "tags": {
+        "type": "jwt",
+        "category": "api_security",
+        "confidence": "0",
+        "module": "business-logic"
+      },
+      "min_version": "1.25.0",
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.jwt",
+                "key_path": [
+                  "header",
+                  "alg"
+                ]
+              }
+            ],
+            "list": [
+              "none",
+              "nonE",
+              "noNe",
+              "noNE",
+              "nOne",
+              "nOnE",
+              "nONe",
+              "nONE",
+              "None",
+              "NonE",
+              "NoNe",
+              "NoNE",
+              "NOne",
+              "NOnE",
+              "NONe",
+              "NONE"
+            ]
+          },
+          "operator": "exact_match"
+        }
+      ],
+      "transformers": [],
+      "output": {
+        "event": false,
+        "keep": true,
+        "attributes": {
+          "_dd.appsec.api.jwt.none_alg": {
+            "value": 1
+          }
+        }
+      }
+    },
+    {
+      "id": "ua0-600-551",
+      "name": "Datadog test scanner - scalar trace-tagging version: user-agent",
+      "tags": {
+        "type": "security_scanner",
+        "category": "attack_attempt",
+        "cwe": "200",
+        "capec": "1000/118/169",
+        "tool_name": "Datadog Canary Test",
+        "confidence": "1",
+        "module": "waf"
+      },
+      "min_version": "1.25.0",
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "user-agent"
+                ]
+              },
+              {
+                "address": "grpc.server.request.metadata",
+                "key_path": [
+                  "dd-canary"
+                ]
+              }
+            ],
+            "regex": "^dd-test-scanner-tag-scalar(?:$|/|\\s)"
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": [],
+      "output": {
+        "event": false,
+        "attributes": {
+          "_dd.appsec.test.scanner.scalar": {
+            "value": 1
+          }
+        }
+      }
+    },
+    {
+      "id": "ua0-600-552",
+      "name": "Datadog test scanner - reference trace-tagging version: user-agent",
+      "tags": {
+        "type": "security_scanner",
+        "category": "attack_attempt",
+        "cwe": "200",
+        "capec": "1000/118/169",
+        "tool_name": "Datadog Canary Test",
+        "confidence": "1",
+        "module": "waf"
+      },
+      "min_version": "1.25.0",
+      "conditions": [
+        {
+          "parameters": {
+            "inputs": [
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "user-agent"
+                ]
+              },
+              {
+                "address": "grpc.server.request.metadata",
+                "key_path": [
+                  "dd-canary"
+                ]
+              }
+            ],
+            "regex": "^dd-test-scanner-tag-ref(?:$|/|\\s)"
+          },
+          "operator": "match_regex"
+        }
+      ],
+      "transformers": [],
+      "output": {
+        "event": false,
+        "attributes": {
+          "_dd.appsec.test.scanner.reference": {
+            "address": "server.request.headers.no_cookies",
+            "key_path": [
+              "user-agent"
+            ]
+          }
+        }
+      }
+    }
+  ],
   "processors": [
     {
       "id": "http-endpoint-fingerprint",
@@ -9074,6 +9389,28 @@
       "evaluate": true,
       "output": true
     },
+    {
+      "id": "decode-auth-jwt",
+      "generator": "jwt_decode",
+      "min_version": "1.25.0",
+      "parameters": {
+        "mappings": [
+          {
+            "inputs": [
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "authorization"
+                ]
+              }
+            ],
+            "output": "server.request.jwt"
+          }
+        ]
+      },
+      "evaluate": true,
+      "output": false
+    },
     {
       "id": "http-network-fingerprint",
       "generator": "http_network_fingerprint",
@@ -9918,6 +10255,24 @@
         "category": "payment"
       }
     },
+    {
+      "id": "c542c147-3883-43d6-a067-178e4a7bd65d",
+      "name": "Password",
+      "key": {
+        "operator": "match_regex",
+        "parameters": {
+          "regex": "\\bpass(?:[_-]?word|wd)?\\b|\\bpwd\\b",
+          "options": {
+            "case_sensitive": false,
+            "min_length": 3
+          }
+        }
+      },
+      "tags": {
+        "type": "password",
+        "category": "credentials"
+      }
+    },
     {
       "id": "18b608bd7a764bff5b2344c0",
       "name": "Phone number",
@@ -10146,4 +10501,4 @@
       }
     }
   ]
-}
+}

data/lib/datadog/appsec/assets/waf_rules/strict.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.14.2"
+    "rules_version": "1.15.1"
   },
   "rules": [
     {
@@ -1746,6 +1746,7 @@
       "transformers": []
     }
   ],
+  "rules_compat": [],
   "processors": [
     {
       "id": "http-endpoint-fingerprint",
@@ -1950,6 +1951,28 @@
       "evaluate": true,
       "output": true
     },
+    {
+      "id": "decode-auth-jwt",
+      "generator": "jwt_decode",
+      "min_version": "1.25.0",
+      "parameters": {
+        "mappings": [
+          {
+            "inputs": [
+              {
+                "address": "server.request.headers.no_cookies",
+                "key_path": [
+                  "authorization"
+                ]
+              }
+            ],
+            "output": "server.request.jwt"
+          }
+        ]
+      },
+      "evaluate": true,
+      "output": false
+    },
     {
       "id": "http-network-fingerprint",
       "generator": "http_network_fingerprint",
@@ -2794,6 +2817,24 @@
         "category": "payment"
       }
     },
+    {
+      "id": "c542c147-3883-43d6-a067-178e4a7bd65d",
+      "name": "Password",
+      "key": {
+        "operator": "match_regex",
+        "parameters": {
+          "regex": "\\bpass(?:[_-]?word|wd)?\\b|\\bpwd\\b",
+          "options": {
+            "case_sensitive": false,
+            "min_length": 3
+          }
+        }
+      },
+      "tags": {
+        "type": "password",
+        "category": "credentials"
+      }
+    },
     {
       "id": "18b608bd7a764bff5b2344c0",
       "name": "Phone number",
@@ -3022,4 +3063,4 @@
       }
     }
   ]
-}
+}

data/lib/datadog/appsec/compressed_json.rb

@@ -8,7 +8,7 @@ require_relative '../core/utils/base64'
 
 module Datadog
   module AppSec
-    # Converts derivative schema payloads into JSON and compresses them into a
+    # Converts complex schema payloads into JSON and compresses them into a
     # base64 encoded string if the payload is worth compressing.
     #
     # See: https://github.com/DataDog/dd-trace-rb/pull/3177#issuecomment-1747221082

data/lib/datadog/appsec/configuration/settings.rb

@@ -352,6 +352,15 @@ module Datadog
                   o.default true
                 end
 
+                settings :endpoint_collection do
+                  # Enables reporting of application routes at application start via telemetry
+                  option :enabled do |o|
+                    o.type :bool, nilable: true
+                    o.env 'DD_API_SECURITY_ENDPOINT_COLLECTION_ENABLED'
+                    o.default true
+                  end
+                end
+
                 # NOTE: Unfortunately, we have to go with Float due to other libs
                 #       setup, even tho we don't plan to support sub-second delays.
                 #

data/lib/datadog/appsec/contrib/active_record/instrumentation.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../event'
+require_relative '../../trace_keeper'
 require_relative '../../security_event'
 
 module Datadog
@@ -31,7 +32,8 @@ module Datadog
             result = context.run_rasp(Ext::RASP_SQLI, {}, ephemeral_data, waf_timeout)
 
             if result.match?
-              AppSec::Event.tag_and_keep!(context, result)
+              AppSec::Event.tag(context, result)
+              TraceKeeper.keep!(context.trace) if result.keep?
 
               context.events.push(
                 AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)

data/lib/datadog/appsec/contrib/excon/ssrf_detection_middleware.rb

@@ -3,6 +3,7 @@
 require 'excon'
 
 require_relative '../../event'
+require_relative '../../trace_keeper'
 require_relative '../../security_event'
 
 module Datadog
@@ -22,7 +23,8 @@ module Datadog
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, Datadog.configuration.appsec.waf_timeout)
 
             if result.match?
-              AppSec::Event.tag_and_keep!(context, result)
+              AppSec::Event.tag(context, result)
+              TraceKeeper.keep!(context.trace) if result.keep?
 
               context.events.push(
                 AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)
@@ -38,4 +40,3 @@ module Datadog
     end
   end
 end
-# rubocop:enable Naming/FileName

data/lib/datadog/appsec/contrib/faraday/ssrf_detection_middleware.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../../event'
+require_relative '../../trace_keeper'
 require_relative '../../security_event'
 
 module Datadog
@@ -21,7 +22,8 @@ module Datadog
             result = context.run_rasp(Ext::RASP_SSRF, {}, ephemeral_data, Datadog.configuration.appsec.waf_timeout)
 
             if result.match?
-              AppSec::Event.tag_and_keep!(context, result)
+              AppSec::Event.tag(context, result)
+              TraceKeeper.keep!(context.trace) if result.keep?
 
               context.events.push(
                 AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)

data/lib/datadog/appsec/contrib/graphql/gateway/watcher.rb

@@ -3,6 +3,7 @@
 require 'json'
 
 require_relative '../../../event'
+require_relative '../../../trace_keeper'
 require_relative '../../../security_event'
 require_relative '../../../instrumentation/gateway'
 
@@ -32,7 +33,8 @@ module Datadog
                     result = context.run_waf(persistent_data, {}, Datadog.configuration.appsec.waf_timeout)
 
                     if result.match?
-                      AppSec::Event.tag_and_keep!(context, result)
+                      AppSec::Event.tag(context, result)
+                      TraceKeeper.keep!(context.trace) if result.keep?
 
                       context.events.push(
                         AppSec::SecurityEvent.new(result, trace: context.trace, span: context.span)