datadog 2.35.0 → 2.37.0

data/lib/datadog/appsec/route_normalizer/route_pattern.rb

@@ -0,0 +1,378 @@
+# frozen_string_literal: true
+
+require_relative 'route_text'
+
+module Datadog
+  module AppSec
+    module RouteNormalizer
+      # Normalizes a route spec pattern into the route format, inspired by
+      # OpenAPI v3 path templating (best effort)
+      #
+      # Example:
+      #
+      #   /users/:id           => /users/{id}
+      #   /photos/:id.:format  => /photos/{id+format}
+      #   /posts/:id(.:format) => /posts/{id+format}
+      #   /files/*path         => /files/{path}
+      #   /hello world         => /hello%20world
+      #
+      # NOTE: When a request path is supplied, optional groups `(...)` are resolved
+      #       against it: a group is kept only when the path actually matched it.
+      #
+      # NOTE: Without a request path (or for paths longer than {MAX_RESOLVE_LENGTH}),
+      #       optional markers are flattened and kept as if present.
+      #
+      # @api private
+      class RoutePattern
+        GROUP_OPEN_CHAR = '('
+        GROUP_CLOSE_CHAR = ')'
+
+        OPTIONAL_GROUP_SUFFIX_CHAR = '?'
+        OPTIONAL_GROUP_SIGILS = [
+          GROUP_OPEN_CHAR,
+          GROUP_CLOSE_CHAR,
+          OPTIONAL_GROUP_SUFFIX_CHAR
+        ].join
+
+        NAMED_PARAM_PREFIX_CHAR = ':'
+        GLOB_PARAM_PREFIX_CHAR = '*'
+
+        PATTERN_STRUCTURE_CHARS = [
+          NAMED_PARAM_PREFIX_CHAR,
+          GLOB_PARAM_PREFIX_CHAR,
+          GROUP_OPEN_CHAR,
+          GROUP_CLOSE_CHAR
+        ].join
+
+        PARAM_NAME_CHARS = [*'a'..'z', *'A'..'Z', *'0'..'9', '_'].join
+
+        EXCLUDED_PARAM_NAME_TERMINATOR_CHARS = [
+          PARAM_NAME_CHARS,
+          PATTERN_STRUCTURE_CHARS
+        ].join
+
+        MAX_RESOLVE_LENGTH = 8192
+
+        Checkpoint = Struct.new(:resolved_length, :pattern_index, :path_index)
+
+        # Param sigils mark where dynamic route syntax may start
+        #
+        # Example:
+        #
+        #   users    -> no
+        #   :id      -> yes
+        #   user-:id -> yes
+        #   *        -> yes
+        #   foo:     -> yes (validated later by {PARAM_TOKENS})
+        PARAM_START_SIGILS = /[:\*]/
+        PARAM_TOKENS = /:\w+|(?<!\w)\*\w*/
+
+        def initialize(pattern)
+          @pattern = pattern
+        end
+
+        def normalize(path: nil)
+          @path_index = 0
+          @pattern_index = 0
+
+          @path = path || ''
+          @path_length = @path.length
+          @pattern_length = @pattern.length
+
+          nameless_counter = 0
+          pattern = resolve_pattern_optionals(path)
+
+          result = pattern.split('/', -1).each_with_object(+'') do |segment, memo|
+            memo << '/' unless memo.empty? && segment.empty?
+            next if segment.empty?
+
+            unless segment.match?(PARAM_START_SIGILS)
+              memo << RouteText.escape(segment)
+              next
+            end
+
+            tokens = segment.scan(PARAM_TOKENS)
+
+            if tokens.empty?
+              memo << RouteText.escape(segment)
+              next
+            end
+
+            names = tokens.map do |token|
+              if token.length > 1
+                token[1..-1]
+              else
+                "param#{nameless_counter += 1}"
+              end
+            end
+
+            memo << "{#{names.join('+')}}"
+          end
+
+          result.start_with?('/') ? result : "/#{result}"
+        end
+
+        private
+
+        # Optional resolution walks the route pattern and request path together
+        #
+        # Example:
+        #
+        #   pattern: /posts/:id(.:format)
+        #   path:    /posts/1.json
+        #                    ^
+        #                    `.` matches, so the optional group is kept
+        #
+        #   pattern: /posts/:id(.:format)
+        #   path:    /posts/1
+        #                   ^
+        #                   path ended before `.`, so the optional group is skipped
+        #
+        # Checkpoints let optional groups fail after a partial match
+        #
+        #   pattern: /posts(/:id)/edit
+        #   path:    /posts/edit
+        #                 ^
+        #                 `(/:id)` starts like `/edit`, so a checkpoint is saved
+        #
+        #   pattern: /posts(/:id)/edit
+        #   path:    /posts/edit
+        #                 ^
+        #                 `:id` cannot consume `edit`, so we rewind after `(/:id)`
+        def resolve_pattern_optionals(path)
+          return @pattern if !@pattern.include?(GROUP_OPEN_CHAR) && !@pattern.include?(OPTIONAL_GROUP_SUFFIX_CHAR)
+
+          unless resolve_pattern_optionals?(path)
+            return remove_optional_group_sigils(@pattern) if @pattern.include?(GROUP_OPEN_CHAR)
+
+            return @pattern
+          end
+
+          resolved = +''
+          checkpoints = []
+
+          while @pattern_index < @pattern_length
+            char = @pattern[@pattern_index] # : String
+
+            case char
+            when GROUP_OPEN_CHAR
+              optional_group_end_index = find_next_optional_group_end_index
+
+              # NOTE: The request path may share the optional group's first char
+              #       with the next required segment
+              #
+              #       Example: `/posts/edit` request path starts like `(/:id)` in pattern
+              #                `/posts(/:id)/edit`, but `:id` must be skipped
+              if @path_index < @path_length && @path[@path_index] == @pattern[@pattern_index + 1]
+                checkpoints << Checkpoint.new(resolved.length, optional_group_end_index + 1, @path_index)
+                @pattern_index += 1
+              else
+                @pattern_index = optional_group_end_index + 1
+              end
+            when GROUP_CLOSE_CHAR
+              @pattern_index += 1
+              @pattern_index += 1 if @pattern[@pattern_index] == OPTIONAL_GROUP_SUFFIX_CHAR
+            when OPTIONAL_GROUP_SUFFIX_CHAR
+              @pattern_index += 1
+            when NAMED_PARAM_PREFIX_CHAR
+              param_name_end_index = find_param_name_end_index(@pattern_index)
+
+              # NOTE: A ':' without a param name is literal text
+              #       Example: `/foo:` must match a literal `:`, not a param
+              if param_name_end_index == @pattern_index + 1
+                unless expected_path_char?(NAMED_PARAM_PREFIX_CHAR)
+                  checkpoint = restore_checkpoint!(resolved, checkpoints)
+                  return remove_optional_group_sigils(@pattern) unless checkpoint
+
+                  @pattern_index = checkpoint.pattern_index
+                  @path_index = checkpoint.path_index
+
+                  next
+                end
+
+                @path_index += 1
+                @pattern_index += 1
+                resolved << NAMED_PARAM_PREFIX_CHAR
+
+                next
+              end
+
+              # NOTE: A trailing `?` makes the param optional
+              #       Example: `/posts/:id.?:format?` with `/posts/1` skips `:format?`
+              if @pattern[param_name_end_index] == OPTIONAL_GROUP_SUFFIX_CHAR && @path_index >= @path_length
+                @pattern_index = param_name_end_index + 1
+                next
+              end
+
+              param_token = @pattern[@pattern_index...param_name_end_index] # : String
+              resolved << param_token
+
+              terminator_char = find_param_value_terminator_char(param_name_end_index: param_name_end_index)
+              @path_index = find_param_value_end_index(terminator_char: terminator_char)
+
+              @pattern_index = param_name_end_index
+            when GLOB_PARAM_PREFIX_CHAR
+              param_name_end_index = find_param_name_end_index(@pattern_index)
+              terminator_char = find_param_value_terminator_char(param_name_end_index: param_name_end_index)
+
+              param_token = @pattern[@pattern_index...param_name_end_index] # : String
+              resolved << param_token
+
+              @path_index = find_glob_value_end_index(terminator_char: terminator_char)
+              @pattern_index = param_name_end_index
+            else
+              unless expected_path_char?(char)
+                if @pattern[@pattern_index + 1] == OPTIONAL_GROUP_SUFFIX_CHAR
+                  @pattern_index += 2
+                  next
+                end
+
+                next if skip_char_with_optional_param?
+
+                checkpoint = restore_checkpoint!(resolved, checkpoints)
+                return remove_optional_group_sigils(@pattern) unless checkpoint
+
+                @pattern_index = checkpoint.pattern_index
+                @path_index = checkpoint.path_index
+
+                next
+              end
+
+              resolved << char
+
+              @path_index += 1
+              @pattern_index += 1
+            end
+          end
+
+          resolved
+        end
+
+        def resolve_pattern_optionals?(path)
+          return false unless path
+
+          @path_length <= MAX_RESOLVE_LENGTH
+        end
+
+        def expected_path_char?(char)
+          @path_index < @path_length && @path[@path_index] == char
+        end
+
+        def restore_checkpoint!(resolved, checkpoints)
+          checkpoint = checkpoints.pop
+          return unless checkpoint
+
+          resolved.slice!(checkpoint.resolved_length..-1)
+          checkpoint
+        end
+
+        def remove_optional_group_sigils(pattern)
+          pattern.delete(OPTIONAL_GROUP_SIGILS)
+        end
+
+        # NOTE: When the path ends before a char that prefixes an optional param,
+        #       skip both the char and that param
+        #
+        #       Example:
+        #         pattern: /posts/:id?
+        #         path:    /posts
+        #         skip:          ^^^^^
+        #
+        #         pattern: /a.:b?
+        #         path:    /a
+        #         skip:      ^^^^
+        def skip_char_with_optional_param?
+          return false if @path_index < @path_length
+
+          param_start_index = @pattern_index + 1
+          return false unless param_start_char?(param_start_index)
+
+          param_name_end_index = find_param_name_end_index(param_start_index)
+          return false unless @pattern[param_name_end_index] == OPTIONAL_GROUP_SUFFIX_CHAR
+
+          @pattern_index = param_name_end_index + 1
+          true
+        end
+
+        def param_start_char?(index)
+          @pattern[index] == NAMED_PARAM_PREFIX_CHAR || @pattern[index] == GLOB_PARAM_PREFIX_CHAR
+        end
+
+        def find_param_name_end_index(param_start_index)
+          current_index = param_start_index + 1
+
+          while current_index < @pattern_length
+            char = @pattern[current_index] # : String
+            break unless PARAM_NAME_CHARS.include?(char)
+
+            current_index += 1
+          end
+
+          current_index
+        end
+
+        def find_param_value_terminator_char(param_name_end_index:)
+          char = @pattern[param_name_end_index]
+
+          if char == GROUP_OPEN_CHAR
+            @pattern[param_name_end_index + 1]
+          else
+            char
+          end
+        end
+
+        def find_param_value_end_index(terminator_char:)
+          terminator_indexes = []
+          slash_index = @path.index('/', @path_index)
+          dot_index = @path.index('.', @path_index)
+
+          terminator_indexes << slash_index if slash_index
+          terminator_indexes << dot_index if dot_index
+
+          if terminator_char && !EXCLUDED_PARAM_NAME_TERMINATOR_CHARS.include?(terminator_char)
+            custom_index = @path.index(terminator_char, @path_index)
+            terminator_indexes << custom_index if custom_index
+          end
+
+          terminator_indexes.empty? ? @path.length : terminator_indexes.min
+        end
+
+        def find_glob_value_end_index(terminator_char:)
+          return @path_length unless terminator_char
+          return @path_length if EXCLUDED_PARAM_NAME_TERMINATOR_CHARS.include?(terminator_char)
+
+          terminator_index = @path.rindex(terminator_char)
+          return @path_length unless terminator_index && terminator_index >= @path_index
+
+          terminator_index
+        end
+
+        def find_next_optional_group_end_index
+          depth = 0
+          index = @pattern_index
+
+          while index < @pattern_length
+            char = @pattern[index]
+            depth += 1 if char == GROUP_OPEN_CHAR
+
+            if char == GROUP_CLOSE_CHAR
+              depth -= 1
+
+              if depth.zero?
+                next_index = index + 1
+
+                return next_index if @pattern[next_index] == OPTIONAL_GROUP_SUFFIX_CHAR
+
+                return index
+              end
+            end
+
+            index += 1
+          end
+
+          index
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/route_normalizer/route_text.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module RouteNormalizer
+      # Percent-encodes route text, leaving param templates untouched
+      #
+      # @api private
+      module RouteText
+        DISALLOWED_CHARS = %r{[^\w.~/-]}
+
+        # Per-byte percent-encoding lookup, indexed by byte value (0-255)
+        #
+        # Example:
+        #
+        #   32 => %20   (space)
+        #   33 => %21   (!)
+        #   47 => /     (passthrough)
+        #   65 => A     (passthrough)
+        #  195 => %C3   (UTF-8 lead byte)
+        #  169 => %A9   (UTF-8 continuation byte)
+        BYTE_ENCODING_TABLE = Array.new(256) do |byte|
+          char = byte.chr
+          char.match?(DISALLOWED_CHARS) ? -('%%%02X' % byte) : -char
+        end.freeze
+
+        # Max bytes one input byte expands to as percent-encoded `%XX`
+        MAX_ENCODED_BYTE_SIZE = 3
+
+        module_function
+
+        # Escapes literal route text into normalized-route form
+        #
+        # Example:
+        #
+        #   a+b => a%2Bb
+        #   café => caf%C3%A9
+        #   /users/path => /users/path
+        #
+        # NOTE: {URI::Parser#escape} with {DISALLOWED_CHARS} gives the same output,
+        #       but the generic gsub/sprintf path is slower and allocates more per request
+        def escape(text)
+          return text unless text.match?(DISALLOWED_CHARS)
+
+          buffer = String.new(capacity: text.bytesize * MAX_ENCODED_BYTE_SIZE, encoding: Encoding::UTF_8)
+          text.each_byte { |byte| buffer << BYTE_ENCODING_TABLE.fetch(byte) }
+          buffer
+        # NOTE: Defensive only — this can never happen. {String#each_byte} yields
+        #       integers in 0-255 and {BYTE_ENCODING_TABLE} has an entry for every one
+        rescue IndexError => e
+          AppSec.telemetry&.report(e, description: 'AppSec: Route text byte outside 0-255 escape table')
+          '~invalid~'
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/route_normalizer.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require_relative 'route_normalizer/route_pattern'
+require_relative 'route_normalizer/rails_route_pattern'
+
+module Datadog
+  module AppSec
+    # Extracts framework route data from Rack env and normalizes it into
+    # OpenAPI v3 compatible route format
+    #
+    # @api private
+    module RouteNormalizer
+      GRAPE_ROUTE_KEY = 'grape.routing_args'
+      SINATRA_ROUTE_KEY = 'sinatra.route'
+      DATADOG_ROUTE_KEY = 'datadog.action_dispatch.route'
+      RAILS_ROUTE_KEY = 'action_dispatch.route'
+      RAILS_ROUTE_URI_PATTERN_KEY = 'action_dispatch.route_uri_pattern'
+      RAILS_PATH_PARAMS_KEY = 'action_dispatch.request.path_parameters'
+
+      module_function
+
+      # Extracts a normalized route path for a Rack request.
+      #
+      # Examples:
+      #
+      #   Grape:   "/api/users/:id"              => "/api/users/{id}"
+      #   Sinatra: "GET /download/*.*"           => "/download/{param1+param2}"
+      #   Rails:   "/posts/:id(.:format)" + path => "/posts/{id+format}"
+      #   Pattern: "/users/:id"                  => "/users/{id}"
+      def extract_normalized_route(rack_env, prefix: nil, pattern: nil)
+        route_prefix = prefix || rack_env['SCRIPT_NAME']
+
+        if rack_env.key?(GRAPE_ROUTE_KEY)
+          route_info = rack_env[GRAPE_ROUTE_KEY][:route_info]
+          normalize_route_string(route_info&.pattern&.origin)
+        elsif rack_env.key?(SINATRA_ROUTE_KEY)
+          normalize_route_string(rack_env[SINATRA_ROUTE_KEY].split(' ', 2)[1])
+        elsif (rails_route = rack_env[DATADOG_ROUTE_KEY] || rack_env[RAILS_ROUTE_KEY])
+          normalize_rails_route(rails_route, rack_env, route_prefix)
+        elsif rack_env.key?(RAILS_ROUTE_URI_PATTERN_KEY)
+          normalize_rails_route(rack_env[RAILS_ROUTE_URI_PATTERN_KEY], rack_env, route_prefix)
+        else
+          normalize_route_string(pattern)
+        end
+      rescue => e
+        AppSec.telemetry&.report(e, description: 'AppSec: Could not compute normalized route')
+
+        nil
+      end
+
+      private_class_method def normalize_rails_route(route, rack_env, route_prefix)
+        path_params = rack_env.fetch(RAILS_PATH_PARAMS_KEY, {})
+        path = path_without_prefix(rack_env['PATH_INFO'].to_s, route_prefix)
+
+        RailsRoutePattern.new(route).normalize(path_params: path_params, path: path)
+      end
+
+      private_class_method def normalize_route_string(route)
+        return unless route
+
+        RoutePattern.new(route).normalize
+      end
+
+      private_class_method def path_without_prefix(path, prefix)
+        return path unless prefix
+
+        prefix = prefix.to_s
+        return path if prefix.empty? || prefix == '/'
+        return path unless path.start_with?(prefix)
+
+        next_char = path[prefix.length]
+        return path if next_char && next_char != '/'
+
+        # NOTE: `start_with?` guards impossible `nil` case
+        stripped = path[prefix.length..-1] # : String
+        stripped.empty? ? '/' : stripped
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/http/body_reader.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Utils
+      module HTTP
+        module BodyReader
+          module_function
+
+          def read(body, limit:, rewind_before_read: false)
+            return body.byteslice(0, limit + 1) if body.is_a?(String)
+            return if body.nil? || !body.respond_to?(:read)
+            return if rewind_before_read && !body.respond_to?(:rewind)
+
+            read_stream(body, limit: limit, rewind_before_read: rewind_before_read)
+          end
+
+          def read_stream(body, limit:, rewind_before_read: false)
+            # @type var rewound: bool
+            rewound = false
+
+            if rewind_before_read
+              return unless rewind(body)
+
+              rewound = true
+            end
+
+            buffer = +''.b
+            max = limit + 1
+
+            while buffer.bytesize <= limit
+              chunk = body.read(max - buffer.bytesize)
+              break if chunk.nil? || chunk.empty?
+
+              buffer << chunk
+            end
+
+            buffer
+          rescue => e
+            Datadog.logger.debug { "AppSec: Failed to read body: #{e.class}: #{e.message}" }
+
+            raise
+          ensure
+            rewind(body) if rewound
+          end
+
+          private_class_method def rewind(body)
+            return false unless body.respond_to?(:rewind)
+
+            # NOTE: Steep can't narrow interfaces via `respond_to?` guard-clause
+            body.rewind # steep:ignore
+            true
+          rescue => e
+            Datadog.logger.debug { "AppSec: Failed to rewind body: #{e.class}: #{e.message}" }
+            false
+          end
+        end
+      end
+    end
+  end
+end