datadog 2.28.0 → 2.41.0

data/lib/datadog/appsec/route_normalizer/rails_route_pattern.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require_relative "route_text"
+require_relative "route_pattern"
+
+module Datadog
+  module AppSec
+    module RouteNormalizer
+      # Normalizes Rails route patterns into route format, inspired by
+      # OpenAPI v3 path templating
+      #
+      # NOTE: Uses the parsed Journey AST when available
+      #
+      # @api private
+      class RailsRoutePattern
+        DOT_CHAR = "."
+        GROUP_OPEN_CHAR = "("
+        OPTIONAL_GROUP_PATTERN = /\(([^()]*)\)/
+        NAMED_PARAM_PREFIX_CHAR = ":"
+        GLOB_PARAM_PREFIX_CHAR = "*"
+
+        def initialize(pattern)
+          @pattern = pattern
+        end
+
+        def normalize(path_params:, path:)
+          @path_params = path_params
+          @path = path
+
+          if @pattern.is_a?(String)
+            # NOTE: Journey groups without route params are never kept
+            #       Example: `/foo(/bar)` with request `/foo/bar` normalizes to `/foo`
+            pattern = remove_paramless_optional_groups(@pattern)
+            return RoutePattern.new(pattern).normalize(path: path)
+          end
+
+          route_path = @pattern.path
+          route_spec = route_path.spec
+
+          unless route_spec.respond_to?(:type)
+            return RoutePattern.new(route_spec.to_s).normalize(path: path)
+          end
+
+          if route_path.names.empty?
+            route_string = route_spec.to_s
+            return RouteText.escape(route_string) unless route_string.include?(GROUP_OPEN_CHAR)
+          end
+
+          buffer = Buffer.new
+          visit_route_node(route_spec, buffer)
+          buffer.to_path
+        end
+
+        private
+
+        def remove_paramless_optional_groups(pattern)
+          result = pattern
+
+          loop do
+            substituted = result.gsub(OPTIONAL_GROUP_PATTERN) do
+              # NOTE: OPTIONAL_GROUP_PATTERN always captures a string for each gsub match
+              group = ::Regexp.last_match(1) # : String
+              optional_group_has_route_params?(group) ? "(#{group})" : ""
+            end
+
+            return result if substituted == result
+
+            result = substituted
+          end
+        end
+
+        def visit_route_node(node, buffer)
+          case node.type
+          when :CAT
+            visit_route_node(node.left, buffer)
+            visit_route_node(node.right, buffer)
+          when :SLASH
+            buffer.flush
+          when :LITERAL
+            buffer.add_literal(node.left)
+          when :DOT
+            buffer.add_literal(DOT_CHAR)
+          when :SYMBOL, :STAR
+            buffer.add_param(node.name)
+          when :GROUP
+            visit_route_node(node.left, buffer) if group_present?(node.left)
+          end
+        end
+
+        def group_present?(node)
+          param_names = collect_group_param_names(node, [])
+          return false if param_names.empty?
+
+          param_names.all? { |name| param_matched_path?(name.to_sym) }
+        end
+
+        def param_matched_path?(name)
+          return @path_params[name].is_a?(String) unless name == :format
+
+          format = @path_params[:format]
+          return false if !format.is_a?(String) || format.empty?
+
+          dot_index = @path.length - format.length - 1
+          return false if dot_index < 0 || @path[dot_index] != DOT_CHAR
+
+          @path.end_with?(format)
+        end
+
+        def collect_group_param_names(node, memo)
+          case node.type
+          when :SYMBOL, :STAR
+            memo << node.name
+          when :CAT
+            collect_group_param_names(node.left, memo)
+            collect_group_param_names(node.right, memo)
+          end
+
+          memo
+        end
+
+        def optional_group_has_route_params?(group)
+          group.include?(NAMED_PARAM_PREFIX_CHAR) || group.include?(GLOB_PARAM_PREFIX_CHAR)
+        end
+
+        class Buffer
+          def initialize
+            @segments = []
+            @text = +""
+            @params = []
+            @nameless_param_count = 0
+          end
+
+          def add_literal(text)
+            @text << text
+          end
+
+          def add_param(name)
+            @params << name
+          end
+
+          def flush
+            return if @text.empty? && @params.empty?
+
+            @segments << if @params.empty?
+              RouteText.escape(@text)
+            else
+              render_params(@params)
+            end
+
+            @text = +""
+            @params.clear
+          end
+
+          def to_path
+            flush
+
+            "/#{@segments.join("/")}"
+          end
+
+          private
+
+          def render_params(params)
+            names = params.map do |name| # $ String
+              next name unless name.empty?
+
+              @nameless_param_count += 1
+              "param#{@nameless_param_count}"
+            end
+
+            "{#{names.join("+")}}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/route_normalizer/route_pattern.rb

@@ -0,0 +1,378 @@
+# frozen_string_literal: true
+
+require_relative "route_text"
+
+module Datadog
+  module AppSec
+    module RouteNormalizer
+      # Normalizes a route spec pattern into the route format, inspired by
+      # OpenAPI v3 path templating (best effort)
+      #
+      # Example:
+      #
+      #   /users/:id           => /users/{id}
+      #   /photos/:id.:format  => /photos/{id+format}
+      #   /posts/:id(.:format) => /posts/{id+format}
+      #   /files/*path         => /files/{path}
+      #   /hello world         => /hello%20world
+      #
+      # NOTE: When a request path is supplied, optional groups `(...)` are resolved
+      #       against it: a group is kept only when the path actually matched it.
+      #
+      # NOTE: Without a request path (or for paths longer than {MAX_RESOLVE_LENGTH}),
+      #       optional markers are flattened and kept as if present.
+      #
+      # @api private
+      class RoutePattern
+        GROUP_OPEN_CHAR = "("
+        GROUP_CLOSE_CHAR = ")"
+
+        OPTIONAL_GROUP_SUFFIX_CHAR = "?"
+        OPTIONAL_GROUP_SIGILS = [
+          GROUP_OPEN_CHAR,
+          GROUP_CLOSE_CHAR,
+          OPTIONAL_GROUP_SUFFIX_CHAR,
+        ].join
+
+        NAMED_PARAM_PREFIX_CHAR = ":"
+        GLOB_PARAM_PREFIX_CHAR = "*"
+
+        PATTERN_STRUCTURE_CHARS = [
+          NAMED_PARAM_PREFIX_CHAR,
+          GLOB_PARAM_PREFIX_CHAR,
+          GROUP_OPEN_CHAR,
+          GROUP_CLOSE_CHAR,
+        ].join
+
+        PARAM_NAME_CHARS = [*"a".."z", *"A".."Z", *"0".."9", "_"].join
+
+        EXCLUDED_PARAM_NAME_TERMINATOR_CHARS = [
+          PARAM_NAME_CHARS,
+          PATTERN_STRUCTURE_CHARS,
+        ].join
+
+        MAX_RESOLVE_LENGTH = 8192
+
+        Checkpoint = Struct.new(:resolved_length, :pattern_index, :path_index)
+
+        # Param sigils mark where dynamic route syntax may start
+        #
+        # Example:
+        #
+        #   users    -> no
+        #   :id      -> yes
+        #   user-:id -> yes
+        #   *        -> yes
+        #   foo:     -> yes (validated later by {PARAM_TOKENS})
+        PARAM_START_SIGILS = /[:\*]/
+        PARAM_TOKENS = /:\w+|(?<!\w)\*\w*/
+
+        def initialize(pattern)
+          @pattern = pattern
+        end
+
+        def normalize(path: nil)
+          @path_index = 0
+          @pattern_index = 0
+
+          @path = path || ""
+          @path_length = @path.length
+          @pattern_length = @pattern.length
+
+          nameless_counter = 0
+          pattern = resolve_pattern_optionals(path)
+
+          result = pattern.split("/", -1).each_with_object(+"") do |segment, memo|
+            memo << "/" unless memo.empty? && segment.empty?
+            next if segment.empty?
+
+            unless segment.match?(PARAM_START_SIGILS)
+              memo << RouteText.escape(segment)
+              next
+            end
+
+            tokens = segment.scan(PARAM_TOKENS)
+
+            if tokens.empty?
+              memo << RouteText.escape(segment)
+              next
+            end
+
+            names = tokens.map do |token|
+              if token.length > 1
+                token[1..-1]
+              else
+                "param#{nameless_counter += 1}"
+              end
+            end
+
+            memo << "{#{names.join("+")}}"
+          end
+
+          result.start_with?("/") ? result : "/#{result}"
+        end
+
+        private
+
+        # Optional resolution walks the route pattern and request path together
+        #
+        # Example:
+        #
+        #   pattern: /posts/:id(.:format)
+        #   path:    /posts/1.json
+        #                    ^
+        #                    `.` matches, so the optional group is kept
+        #
+        #   pattern: /posts/:id(.:format)
+        #   path:    /posts/1
+        #                   ^
+        #                   path ended before `.`, so the optional group is skipped
+        #
+        # Checkpoints let optional groups fail after a partial match
+        #
+        #   pattern: /posts(/:id)/edit
+        #   path:    /posts/edit
+        #                 ^
+        #                 `(/:id)` starts like `/edit`, so a checkpoint is saved
+        #
+        #   pattern: /posts(/:id)/edit
+        #   path:    /posts/edit
+        #                 ^
+        #                 `:id` cannot consume `edit`, so we rewind after `(/:id)`
+        def resolve_pattern_optionals(path)
+          return @pattern if !@pattern.include?(GROUP_OPEN_CHAR) && !@pattern.include?(OPTIONAL_GROUP_SUFFIX_CHAR)
+
+          unless resolve_pattern_optionals?(path)
+            return remove_optional_group_sigils(@pattern) if @pattern.include?(GROUP_OPEN_CHAR)
+
+            return @pattern
+          end
+
+          resolved = +""
+          checkpoints = []
+
+          while @pattern_index < @pattern_length
+            char = @pattern[@pattern_index] # : String
+
+            case char
+            when GROUP_OPEN_CHAR
+              optional_group_end_index = find_next_optional_group_end_index
+
+              # NOTE: The request path may share the optional group's first char
+              #       with the next required segment
+              #
+              #       Example: `/posts/edit` request path starts like `(/:id)` in pattern
+              #                `/posts(/:id)/edit`, but `:id` must be skipped
+              if @path_index < @path_length && @path[@path_index] == @pattern[@pattern_index + 1]
+                checkpoints << Checkpoint.new(resolved.length, optional_group_end_index + 1, @path_index)
+                @pattern_index += 1
+              else
+                @pattern_index = optional_group_end_index + 1
+              end
+            when GROUP_CLOSE_CHAR
+              @pattern_index += 1
+              @pattern_index += 1 if @pattern[@pattern_index] == OPTIONAL_GROUP_SUFFIX_CHAR
+            when OPTIONAL_GROUP_SUFFIX_CHAR
+              @pattern_index += 1
+            when NAMED_PARAM_PREFIX_CHAR
+              param_name_end_index = find_param_name_end_index(@pattern_index)
+
+              # NOTE: A ':' without a param name is literal text
+              #       Example: `/foo:` must match a literal `:`, not a param
+              if param_name_end_index == @pattern_index + 1
+                unless expected_path_char?(NAMED_PARAM_PREFIX_CHAR)
+                  checkpoint = restore_checkpoint!(resolved, checkpoints)
+                  return remove_optional_group_sigils(@pattern) unless checkpoint
+
+                  @pattern_index = checkpoint.pattern_index
+                  @path_index = checkpoint.path_index
+
+                  next
+                end
+
+                @path_index += 1
+                @pattern_index += 1
+                resolved << NAMED_PARAM_PREFIX_CHAR
+
+                next
+              end
+
+              # NOTE: A trailing `?` makes the param optional
+              #       Example: `/posts/:id.?:format?` with `/posts/1` skips `:format?`
+              if @pattern[param_name_end_index] == OPTIONAL_GROUP_SUFFIX_CHAR && @path_index >= @path_length
+                @pattern_index = param_name_end_index + 1
+                next
+              end
+
+              param_token = @pattern[@pattern_index...param_name_end_index] # : String
+              resolved << param_token
+
+              terminator_char = find_param_value_terminator_char(param_name_end_index: param_name_end_index)
+              @path_index = find_param_value_end_index(terminator_char: terminator_char)
+
+              @pattern_index = param_name_end_index
+            when GLOB_PARAM_PREFIX_CHAR
+              param_name_end_index = find_param_name_end_index(@pattern_index)
+              terminator_char = find_param_value_terminator_char(param_name_end_index: param_name_end_index)
+
+              param_token = @pattern[@pattern_index...param_name_end_index] # : String
+              resolved << param_token
+
+              @path_index = find_glob_value_end_index(terminator_char: terminator_char)
+              @pattern_index = param_name_end_index
+            else
+              unless expected_path_char?(char)
+                if @pattern[@pattern_index + 1] == OPTIONAL_GROUP_SUFFIX_CHAR
+                  @pattern_index += 2
+                  next
+                end
+
+                next if skip_char_with_optional_param?
+
+                checkpoint = restore_checkpoint!(resolved, checkpoints)
+                return remove_optional_group_sigils(@pattern) unless checkpoint
+
+                @pattern_index = checkpoint.pattern_index
+                @path_index = checkpoint.path_index
+
+                next
+              end
+
+              resolved << char
+
+              @path_index += 1
+              @pattern_index += 1
+            end
+          end
+
+          resolved
+        end
+
+        def resolve_pattern_optionals?(path)
+          return false unless path
+
+          @path_length <= MAX_RESOLVE_LENGTH
+        end
+
+        def expected_path_char?(char)
+          @path_index < @path_length && @path[@path_index] == char
+        end
+
+        def restore_checkpoint!(resolved, checkpoints)
+          checkpoint = checkpoints.pop
+          return unless checkpoint
+
+          resolved.slice!(checkpoint.resolved_length..-1)
+          checkpoint
+        end
+
+        def remove_optional_group_sigils(pattern)
+          pattern.delete(OPTIONAL_GROUP_SIGILS)
+        end
+
+        # NOTE: When the path ends before a char that prefixes an optional param,
+        #       skip both the char and that param
+        #
+        #       Example:
+        #         pattern: /posts/:id?
+        #         path:    /posts
+        #         skip:          ^^^^^
+        #
+        #         pattern: /a.:b?
+        #         path:    /a
+        #         skip:      ^^^^
+        def skip_char_with_optional_param?
+          return false if @path_index < @path_length
+
+          param_start_index = @pattern_index + 1
+          return false unless param_start_char?(param_start_index)
+
+          param_name_end_index = find_param_name_end_index(param_start_index)
+          return false unless @pattern[param_name_end_index] == OPTIONAL_GROUP_SUFFIX_CHAR
+
+          @pattern_index = param_name_end_index + 1
+          true
+        end
+
+        def param_start_char?(index)
+          @pattern[index] == NAMED_PARAM_PREFIX_CHAR || @pattern[index] == GLOB_PARAM_PREFIX_CHAR
+        end
+
+        def find_param_name_end_index(param_start_index)
+          current_index = param_start_index + 1
+
+          while current_index < @pattern_length
+            char = @pattern[current_index] # : String
+            break unless PARAM_NAME_CHARS.include?(char)
+
+            current_index += 1
+          end
+
+          current_index
+        end
+
+        def find_param_value_terminator_char(param_name_end_index:)
+          char = @pattern[param_name_end_index]
+
+          if char == GROUP_OPEN_CHAR
+            @pattern[param_name_end_index + 1]
+          else
+            char
+          end
+        end
+
+        def find_param_value_end_index(terminator_char:)
+          terminator_indexes = []
+          slash_index = @path.index("/", @path_index)
+          dot_index = @path.index(".", @path_index)
+
+          terminator_indexes << slash_index if slash_index
+          terminator_indexes << dot_index if dot_index
+
+          if terminator_char && !EXCLUDED_PARAM_NAME_TERMINATOR_CHARS.include?(terminator_char)
+            custom_index = @path.index(terminator_char, @path_index)
+            terminator_indexes << custom_index if custom_index
+          end
+
+          terminator_indexes.empty? ? @path.length : terminator_indexes.min
+        end
+
+        def find_glob_value_end_index(terminator_char:)
+          return @path_length unless terminator_char
+          return @path_length if EXCLUDED_PARAM_NAME_TERMINATOR_CHARS.include?(terminator_char)
+
+          terminator_index = @path.rindex(terminator_char)
+          return @path_length unless terminator_index && terminator_index >= @path_index
+
+          terminator_index
+        end
+
+        def find_next_optional_group_end_index
+          depth = 0
+          index = @pattern_index
+
+          while index < @pattern_length
+            char = @pattern[index]
+            depth += 1 if char == GROUP_OPEN_CHAR
+
+            if char == GROUP_CLOSE_CHAR
+              depth -= 1
+
+              if depth.zero?
+                next_index = index + 1
+
+                return next_index if @pattern[next_index] == OPTIONAL_GROUP_SUFFIX_CHAR
+
+                return index
+              end
+            end
+
+            index += 1
+          end
+
+          index
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/route_normalizer/route_text.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module RouteNormalizer
+      # Percent-encodes route text, leaving param templates untouched
+      #
+      # @api private
+      module RouteText
+        DISALLOWED_CHARS = %r{[^\w.~/-]}
+
+        # Per-byte percent-encoding lookup, indexed by byte value (0-255)
+        #
+        # Example:
+        #
+        #   32 => %20   (space)
+        #   33 => %21   (!)
+        #   47 => /     (passthrough)
+        #   65 => A     (passthrough)
+        #  195 => %C3   (UTF-8 lead byte)
+        #  169 => %A9   (UTF-8 continuation byte)
+        BYTE_ENCODING_TABLE = Array.new(256) do |byte|
+          char = byte.chr
+          char.match?(DISALLOWED_CHARS) ? -("%%%02X" % byte) : -char
+        end.freeze
+
+        # Max bytes one input byte expands to as percent-encoded `%XX`
+        MAX_ENCODED_BYTE_SIZE = 3
+
+        module_function
+
+        # Escapes literal route text into normalized-route form
+        #
+        # Example:
+        #
+        #   a+b => a%2Bb
+        #   café => caf%C3%A9
+        #   /users/path => /users/path
+        #
+        # NOTE: {URI::Parser#escape} with {DISALLOWED_CHARS} gives the same output,
+        #       but the generic gsub/sprintf path is slower and allocates more per request
+        def escape(text)
+          return text unless text.match?(DISALLOWED_CHARS)
+
+          buffer = String.new(capacity: text.bytesize * MAX_ENCODED_BYTE_SIZE, encoding: Encoding::UTF_8)
+          text.each_byte { |byte| buffer << BYTE_ENCODING_TABLE.fetch(byte) }
+          buffer
+        # NOTE: Defensive only — this can never happen. {String#each_byte} yields
+        #       integers in 0-255 and {BYTE_ENCODING_TABLE} has an entry for every one
+        rescue IndexError => e
+          AppSec.telemetry&.report(e, description: "AppSec: Route text byte outside 0-255 escape table")
+          "~invalid~"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/route_normalizer.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require_relative "route_normalizer/route_pattern"
+require_relative "route_normalizer/rails_route_pattern"
+
+module Datadog
+  module AppSec
+    # Extracts framework route data from Rack env and normalizes it into
+    # OpenAPI v3 compatible route format
+    #
+    # @api private
+    module RouteNormalizer
+      GRAPE_ROUTE_KEY = "grape.routing_args"
+      SINATRA_ROUTE_KEY = "sinatra.route"
+      DATADOG_ROUTE_KEY = "datadog.action_dispatch.route"
+      RAILS_ROUTE_KEY = "action_dispatch.route"
+      RAILS_ROUTE_URI_PATTERN_KEY = "action_dispatch.route_uri_pattern"
+      RAILS_PATH_PARAMS_KEY = "action_dispatch.request.path_parameters"
+
+      module_function
+
+      # Extracts a normalized route path for a Rack request.
+      #
+      # Examples:
+      #
+      #   Grape:   "/api/users/:id"              => "/api/users/{id}"
+      #   Sinatra: "GET /download/*.*"           => "/download/{param1+param2}"
+      #   Rails:   "/posts/:id(.:format)" + path => "/posts/{id+format}"
+      #   Pattern: "/users/:id"                  => "/users/{id}"
+      def extract_normalized_route(rack_env, prefix: nil, pattern: nil)
+        route_prefix = prefix || rack_env["SCRIPT_NAME"]
+
+        if rack_env.key?(GRAPE_ROUTE_KEY)
+          route_info = rack_env[GRAPE_ROUTE_KEY][:route_info]
+          normalize_route_string(route_info&.pattern&.origin)
+        elsif rack_env.key?(SINATRA_ROUTE_KEY)
+          normalize_route_string(rack_env[SINATRA_ROUTE_KEY].split(" ", 2)[1])
+        elsif (rails_route = rack_env[DATADOG_ROUTE_KEY] || rack_env[RAILS_ROUTE_KEY])
+          normalize_rails_route(rails_route, rack_env, route_prefix)
+        elsif rack_env.key?(RAILS_ROUTE_URI_PATTERN_KEY)
+          normalize_rails_route(rack_env[RAILS_ROUTE_URI_PATTERN_KEY], rack_env, route_prefix)
+        else
+          normalize_route_string(pattern)
+        end
+      rescue => e
+        AppSec.telemetry&.report(e, description: "AppSec: Could not compute normalized route")
+
+        nil
+      end
+
+      private_class_method def normalize_rails_route(route, rack_env, route_prefix)
+        path_params = rack_env.fetch(RAILS_PATH_PARAMS_KEY, {})
+        path = path_without_prefix(rack_env["PATH_INFO"].to_s, route_prefix)
+
+        RailsRoutePattern.new(route).normalize(path_params: path_params, path: path)
+      end
+
+      private_class_method def normalize_route_string(route)
+        return unless route
+
+        RoutePattern.new(route).normalize
+      end
+
+      private_class_method def path_without_prefix(path, prefix)
+        return path unless prefix
+
+        prefix = prefix.to_s
+        return path if prefix.empty? || prefix == "/"
+        return path unless path.start_with?(prefix)
+
+        next_char = path[prefix.length]
+        return path if next_char && next_char != "/"
+
+        # NOTE: `start_with?` guards impossible `nil` case
+        stripped = path[prefix.length..-1] # : String
+        stripped.empty? ? "/" : stripped
+      end
+    end
+  end
+end