datadog 2.28.0 → 2.41.0

data/lib/datadog/appsec/security_engine/engine.rb

@@ -6,7 +6,7 @@ module Datadog
       # SecurityEngine::Engine creates WAF builder and manages its configuration.
       # It also rebuilds WAF handle from the WAF builder when configuration changes.
       class Engine
-        DEFAULT_RULES_CONFIG_PATH = 'ASM_DD/default'
+        DEFAULT_RULES_CONFIG_PATH = "ASM_DD/default"
         TELEMETRY_ACTIONS = %w[init update].freeze
         DIAGNOSTICS_CONFIG_KEYS = %w[
           rules
@@ -31,21 +31,25 @@ module Datadog
           @waf_builder = WAF::HandleBuilder.new(
             obfuscator: {
               key_regex: appsec_settings.obfuscator_key_regex,
-              value_regex: appsec_settings.obfuscator_value_regex
+              value_regex: appsec_settings.obfuscator_value_regex,
             }
           )
 
           diagnostics = load_default_config(telemetry: telemetry)
-          report_configuration_diagnostics(diagnostics, action: 'init', telemetry: telemetry)
-          @ruleset_version = diagnostics['ruleset_version']
+          report_configuration_diagnostics(diagnostics, action: "init", telemetry: telemetry)
+          @ruleset_version = diagnostics["ruleset_version"]
 
           @handle_ref = ThreadSafeRef.new(@waf_builder.build_handle)
+
+          metric("init", success: true, ruleset_version: @ruleset_version, telemetry: telemetry)
         rescue WAF::Error => e
           error_message = "AppSec security engine failed to initialize"
 
           Datadog.logger.error("#{error_message}, error #{e.inspect}")
           telemetry.report(e, description: error_message)
 
+          metric("init", success: false, ruleset_version: @ruleset_version, telemetry: telemetry)
+
           raise e
         end
 
@@ -54,23 +58,23 @@ module Datadog
         end
 
         def add_or_update_config(config, path:)
-          is_ruleset_update = path.include?('ASM_DD')
+          is_ruleset_update = path.include?("ASM_DD")
 
           # default config has to be removed when adding an ASM_DD config
           remove_config_at_path(DEFAULT_RULES_CONFIG_PATH) if is_ruleset_update
 
           diagnostics = @waf_builder.add_or_update_config(config, path: path)
-          @reconfigured_ruleset_version = diagnostics['ruleset_version'] if diagnostics.key?('ruleset_version')
-          report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
+          @reconfigured_ruleset_version = diagnostics["ruleset_version"] if diagnostics.key?("ruleset_version")
+          report_configuration_diagnostics(diagnostics, action: "update", telemetry: AppSec.telemetry)
 
           # we need to load default config if diagnostics contains top-level error for rules or processors
           if is_ruleset_update &&
-              (diagnostics.key?('error') ||
-              diagnostics.dig('rules', 'error') ||
-              diagnostics.dig('processors', 'errors'))
+              (diagnostics.key?("error") ||
+              diagnostics.dig("rules", "error") ||
+              diagnostics.dig("processors", "errors"))
             diagnostics = load_default_config(telemetry: AppSec.telemetry)
-            @reconfigured_ruleset_version = diagnostics['ruleset_version']
-            report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
+            @reconfigured_ruleset_version = diagnostics["ruleset_version"]
+            report_configuration_diagnostics(diagnostics, action: "update", telemetry: AppSec.telemetry)
           end
 
           diagnostics
@@ -84,10 +88,10 @@ module Datadog
         def remove_config_at_path(path)
           result = @waf_builder.remove_config_at_path(path)
 
-          if result && path != DEFAULT_RULES_CONFIG_PATH && path.include?('ASM_DD')
+          if result && path != DEFAULT_RULES_CONFIG_PATH && path.include?("ASM_DD")
             diagnostics = load_default_config(telemetry: AppSec.telemetry)
-            @reconfigured_ruleset_version = diagnostics['ruleset_version']
-            report_configuration_diagnostics(diagnostics, action: 'update', telemetry: AppSec.telemetry)
+            @reconfigured_ruleset_version = diagnostics["ruleset_version"]
+            report_configuration_diagnostics(diagnostics, action: "update", telemetry: AppSec.telemetry)
           end
 
           result
@@ -103,6 +107,8 @@ module Datadog
           @ruleset_version = @reconfigured_ruleset_version
 
           @handle_ref.current = new_waf_handle
+
+          metric("updates", success: true, ruleset_version: @ruleset_version, telemetry: AppSec.telemetry)
         rescue WAF::Error => e
           # WAF::Error can only be raised during new WAF handle creation or when reading known addresses.
           # This means that the current WAF handle was not yet substituted.
@@ -110,62 +116,77 @@ module Datadog
 
           Datadog.logger.error("#{error_message}, error #{e.inspect}")
           AppSec.telemetry.report(e, description: error_message)
+
+          metric("updates", success: false, ruleset_version: @reconfigured_ruleset_version, telemetry: AppSec.telemetry)
         end
 
         private
 
+        def metric(metric_name, success:, ruleset_version:, telemetry:)
+          telemetry.inc(
+            Ext::TELEMETRY_METRICS_NAMESPACE,
+            "waf.#{metric_name}",
+            1,
+            tags: {
+              waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
+              event_rules_version: ruleset_version.to_s,
+              success: success.to_s,
+            }
+          )
+        end
+
         def load_default_config(telemetry:)
           config = AppSec::Processor::RuleLoader.load_rules(telemetry: telemetry, ruleset: @default_ruleset)
 
           # deprecated - ip and user id denylists should be configured via RC
-          config['rules_data'] ||= AppSec::Processor::RuleLoader.load_data(
+          config["rules_data"] ||= AppSec::Processor::RuleLoader.load_data(
             ip_denylist: @default_ip_denylist,
             user_id_denylist: @default_user_id_denylist
           )
 
           # deprecated - ip passlist should be configured via RC
-          config['exclusions'] ||= AppSec::Processor::RuleLoader.load_exclusions(ip_passlist: @default_ip_passlist)
+          config["exclusions"] ||= AppSec::Processor::RuleLoader.load_exclusions(ip_passlist: @default_ip_passlist)
 
           @waf_builder.add_or_update_config(config, path: DEFAULT_RULES_CONFIG_PATH)
         end
 
         def report_configuration_diagnostics(diagnostics, action:, telemetry:)
-          raise ArgumentError, 'action must be one of TELEMETRY_ACTIONS' unless TELEMETRY_ACTIONS.include?(action)
+          raise ArgumentError, "action must be one of TELEMETRY_ACTIONS" unless TELEMETRY_ACTIONS.include?(action)
 
           common_tags = {
             waf_version: Datadog::AppSec::WAF::VERSION::BASE_STRING,
-            event_rules_version: diagnostics['ruleset_version'].to_s,
-            action: action
+            event_rules_version: diagnostics["ruleset_version"].to_s,
+            action: action,
           }
 
-          if diagnostics['error']
+          if diagnostics["error"]
             telemetry.inc(
-              Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', 1,
-              tags: common_tags.merge(scope: 'top-level')
+              Ext::TELEMETRY_METRICS_NAMESPACE, "waf.config_errors", 1,
+              tags: common_tags.merge(scope: "top-level")
             )
 
-            telemetry.error(diagnostics['error'])
+            telemetry.error(diagnostics["error"])
           end
 
           diagnostics.each do |config_key, config_diagnostics|
             next unless DIAGNOSTICS_CONFIG_KEYS.include?(config_key)
-            next if !config_diagnostics.key?('error') && config_diagnostics.fetch('errors', []).empty?
+            next if !config_diagnostics.key?("error") && config_diagnostics.fetch("errors", []).empty?
 
-            if config_diagnostics['error']
-              telemetry.error(config_diagnostics['error'])
+            if config_diagnostics["error"]
+              telemetry.error(config_diagnostics["error"])
 
               telemetry.inc(
-                Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', 1,
-                tags: common_tags.merge(config_key: config_key, scope: 'top-level')
+                Ext::TELEMETRY_METRICS_NAMESPACE, "waf.config_errors", 1,
+                tags: common_tags.merge(config_key: config_key, scope: "top-level")
               )
-            elsif config_diagnostics['errors']
-              config_diagnostics['errors'].each do |error, config_ids|
+            elsif config_diagnostics["errors"]
+              config_diagnostics["errors"].each do |error, config_ids|
                 telemetry.error("#{error}: [#{config_ids.join(",")}]")
               end
 
               telemetry.inc(
-                Ext::TELEMETRY_METRICS_NAMESPACE, 'waf.config_errors', config_diagnostics['errors'].count,
-                tags: common_tags.merge(config_key: config_key, scope: 'item')
+                Ext::TELEMETRY_METRICS_NAMESPACE, "waf.config_errors", config_diagnostics["errors"].count,
+                tags: common_tags.merge(config_key: config_key, scope: "item")
               )
             end
           end

data/lib/datadog/appsec/security_engine/runner.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'result'
+require_relative "result"
 
 module Datadog
   module AppSec
@@ -79,8 +79,8 @@ module Datadog
         def try_run(persistent_data, ephemeral_data, timeout)
           waf_context.run(persistent_data, ephemeral_data, timeout)
         rescue WAF::LibDDWAFError => e
-          Datadog.logger.debug { "#{@debug_tag} execution error: #{e} backtrace: #{e.backtrace&.first(3)}" }
-          AppSec.telemetry.report(e, description: 'libddwaf-rb internal low-level error')
+          Datadog.logger.debug { "#{@debug_tag} execution error: #{e.class}: #{e.message} backtrace: #{e.backtrace&.first(3)}" }
+          AppSec.telemetry.report(e, description: "libddwaf-rb internal low-level error")
 
           WAF::Result.new(
             status: :err_internal,

data/lib/datadog/appsec/security_event.rb

@@ -6,8 +6,8 @@ module Datadog
     # representing an attack or fingerprinting results as attributes or an API
     # security check with extracted schema.
     class SecurityEvent
-      SCHEMA_KEY_PREFIX = '_dd.appsec.s.'
-      FINGERPRINT_KEY_PREFIX = '_dd.appsec.fp.'
+      SCHEMA_KEY_PREFIX = "_dd.appsec.s."
+      FINGERPRINT_KEY_PREFIX = "_dd.appsec.fp."
 
       attr_reader :waf_result, :trace, :span
 

data/lib/datadog/appsec/trace_keeper.rb

@@ -7,16 +7,28 @@ module Datadog
       def self.keep!(trace)
         return unless trace
 
+        previous_dm = trace.get_tag(Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER)
+
         # NOTE: This action will not set correct decision maker value, so the
         #       trace keeping must be done with additional steps below
         trace.keep!
 
-        # Propagate to downstream services the information that
-        # the current distributed trace is containing at least one ASM event.
-        trace.set_tag(
-          Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
-          Tracing::Sampling::Ext::Decision::ASM
-        )
+        # NOTE: Preserve decision maker if already set by another product.
+        #       As `trace.keep!` resets `_dd.p.dm` to `MANUAL`, we restore the
+        #       previous value when another product has already claimed the
+        #       decision maker.
+        if previous_dm.nil? || previous_dm == Tracing::Sampling::Ext::Decision::MANUAL
+          trace.set_tag(
+            Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
+            Tracing::Sampling::Ext::Decision::ASM
+          )
+        else
+          trace.set_tag(
+            Tracing::Metadata::Ext::Distributed::TAG_DECISION_MAKER,
+            previous_dm,
+          )
+        end
+
         trace.set_distributed_source(Ext::PRODUCT_BIT)
       end
     end

data/lib/datadog/appsec/utils/http/body.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "json"
+require "cgi"
+
+require_relative "url_encoded"
+
+module Datadog
+  module AppSec
+    module Utils
+      module HTTP
+        # Module for handling HTTP body parsing
+        module Body
+          # Matches Rack's default query bytesize limit
+          DEFAULT_BYTESIZE_LIMIT = 4 * 1024 * 1024
+
+          def self.parse(body, media_type:, limit: DEFAULT_BYTESIZE_LIMIT)
+            return if body.nil?
+
+            body.rewind if body.respond_to?(:rewind) # steep:ignore NoMethod
+            # @type var content: ::String?
+            content = body.respond_to?(:read) ? body.read : body # steep:ignore NoMethod, IncompatibleAssignment
+            body.rewind if body.respond_to?(:rewind) # steep:ignore NoMethod
+
+            return if content.nil? || content.empty?
+
+            if media_type.subtype == "json" || media_type.subtype.end_with?("+json")
+              JSON.parse(content)
+            elsif media_type.subtype == "x-www-form-urlencoded"
+              URLEncoded.parse(content, limit: limit)
+            end
+          rescue => e
+            AppSec.telemetry.report(e, description: "AppSec: Failed to parse body")
+
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/http/body_reader.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Datadog
+  module AppSec
+    module Utils
+      module HTTP
+        module BodyReader
+          module_function
+
+          def read(body, limit:, rewind_before_read: false)
+            return body.byteslice(0, limit + 1) if body.is_a?(String)
+            return if body.nil? || !body.respond_to?(:read)
+            return if rewind_before_read && !body.respond_to?(:rewind)
+
+            read_stream(body, limit: limit, rewind_before_read: rewind_before_read)
+          end
+
+          def read_stream(body, limit:, rewind_before_read: false)
+            # @type var rewound: bool
+            rewound = false
+
+            if rewind_before_read
+              return unless rewind(body)
+
+              rewound = true
+            end
+
+            buffer = +"".b
+            max = limit + 1
+
+            while buffer.bytesize <= limit
+              chunk = body.read(max - buffer.bytesize)
+              break if chunk.nil? || chunk.empty?
+
+              buffer << chunk
+            end
+
+            buffer
+          rescue => e
+            Datadog.logger.debug { "AppSec: Failed to read body: #{e.class}: #{e.message}" }
+
+            raise
+          ensure
+            rewind(body) if rewound
+          end
+
+          private_class_method def rewind(body)
+            return false unless body.respond_to?(:rewind)
+
+            # NOTE: Steep can't narrow interfaces via `respond_to?` guard-clause
+            body.rewind # steep:ignore
+            true
+          rescue => e
+            Datadog.logger.debug { "AppSec: Failed to rewind body: #{e.class}: #{e.message}" }
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/appsec/utils/http/media_range.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative 'media_type'
+require_relative "media_type"
 
 module Datadog
   module AppSec
@@ -11,7 +11,7 @@ module Datadog
           class ParseError < ::StandardError
           end
 
-          WILDCARD = '*'
+          WILDCARD = "*"
           WILDCARD_RE = ::Regexp.escape(WILDCARD)
 
           # See: https://www.rfc-editor.org/rfc/rfc7230#section-3.2.6
@@ -96,40 +96,40 @@ module Datadog
 
             raise ParseError, media_range.inspect if media_range_match.nil?
 
-            @type = (media_range_match['type'] || WILDCARD).downcase
-            @subtype = (media_range_match['subtype'] || WILDCARD).downcase
-            @quality = (media_range_match['quality'] || 1.0).to_f
+            @type = (media_range_match["type"] || WILDCARD).downcase
+            @subtype = (media_range_match["subtype"] || WILDCARD).downcase
+            @quality = (media_range_match["quality"] || 1.0).to_f
             @parameters = {}
             @accept_ext = {}
 
-            parameters = media_range_match['parameters']
+            parameters = media_range_match["parameters"]
 
             return if parameters.nil?
 
-            parameters.split(';').map(&:strip).each do |parameter|
+            parameters.split(";").map(&:strip).each do |parameter|
               parameter_match = PARAMETER_RE.match(parameter)
 
               next if parameter_match.nil?
 
-              parameter_name = parameter_match['parameter_name']
-              parameter_value = parameter_match['parameter_value']
+              parameter_name = parameter_match["parameter_name"]
+              parameter_value = parameter_match["parameter_value"]
 
               next if parameter_name.nil? || parameter_value.nil?
 
               @parameters[parameter_name.downcase] = parameter_value.downcase
             end
 
-            accept_exts = media_range_match['accept_exts']
+            accept_exts = media_range_match["accept_exts"]
 
             return if accept_exts.nil?
 
-            accept_exts.split(';').map(&:strip).each do |ext|
+            accept_exts.split(";").map(&:strip).each do |ext|
               ext_match = ACCEPT_EXT_RE.match(ext)
 
               next if ext_match.nil?
 
-              ext_name = ext_match['ext_name']
-              ext_value = ext_match['ext_value']
+              ext_name = ext_match["ext_name"]
+              ext_value = ext_match["ext_value"]
 
               next if ext_name.nil? || ext_value.nil?
 
@@ -168,7 +168,8 @@ module Datadog
           #
           # returns true if the MediaType is accepted by this MediaRange
           def ===(other)
-            return self === MediaType.new(other) if other.is_a?(::String)
+            return false if other.nil?
+            return self === MediaType.parse(other) if other.is_a?(::String)
 
             type == other.type && subtype == other.subtype && other.parameters.all? { |k, v| parameters[k] == v } ||
               type == other.type && wildcard?(:subtype) ||
@@ -188,9 +189,9 @@ module Datadog
           def to_s
             s = +"#{@type}/#{@subtype}"
 
-            s << ';' << @parameters.map { |k, v| "#{k}=#{v}" }.join(';') if @parameters.count > 0
+            s << ";" << @parameters.map { |k, v| "#{k}=#{v}" }.join(";") if @parameters.count > 0
             s << ";q=#{@quality}" if @quality < 1.0
-            s << ';' << @accept_ext.map { |k, v| "#{k}=#{v}" }.join(';') if @accept_ext.count > 0
+            s << ";" << @accept_ext.map { |k, v| "#{k}=#{v}" }.join(";") if @accept_ext.count > 0
 
             s
           end

data/lib/datadog/appsec/utils/http/media_type.rb

@@ -10,9 +10,7 @@ module Datadog
         # - https://www.rfc-editor.org/rfc/rfc7231#section-5.3.1
         # - https://www.rfc-editor.org/rfc/rfc7231#section-5.3.2
         class MediaType
-          ParseError = Class.new(StandardError) # steep:ignore IncompatibleAssignment
-
-          WILDCARD = '*'
+          WILDCARD = "*"
 
           # See: https://www.rfc-editor.org/rfc/rfc7230#section-3.2.6
           TOKEN_RE = /[-#$%&'*+.^_`|~A-Za-z0-9]+/.freeze
@@ -45,48 +43,42 @@ module Datadog
 
           attr_reader :type, :subtype, :parameters
 
-          def self.json?(media_type)
-            return false if media_type.nil? || media_type.empty?
-
-            match = MEDIA_TYPE_RE.match(media_type)
-            return false if match.nil?
+          def self.parse(media)
+            match = MEDIA_TYPE_RE.match(media)
+            return if match.nil?
 
-            subtype = match['subtype']
-            return false if subtype.nil? || subtype.empty?
+            type = match["type"] || WILDCARD
+            type.downcase!
 
+            subtype = match["subtype"] || WILDCARD
             subtype.downcase!
-            subtype == 'json' || subtype.end_with?('+json')
-          end
-
-          def initialize(media_type)
-            match = MEDIA_TYPE_RE.match(media_type)
-            raise ParseError, media_type.inspect if match.nil?
 
-            @type = match['type'] || WILDCARD
-            @type.downcase!
+            parameters = {}
+            params = match["parameters"]
 
-            @subtype = match['subtype'] || WILDCARD
-            @subtype.downcase!
+            unless params.nil? || params.empty?
+              params.scan(PARAMETER_RE) do |name, unquoted_value, quoted_value|
+                # NOTE: Order of unquoted_value and quoted_value does not matter,
+                #       as they are mutually exclusive by the regex.
+                # @type var value: ::String?
+                value = unquoted_value || quoted_value
+                next if name.nil? || value.nil?
 
-            @parameters = {}
+                # See https://github.com/soutaro/steep/issues/2051
+                name.downcase! # steep:ignore NoMethod
+                value.downcase!
 
-            parameters = match['parameters']
-            return if parameters.nil? || parameters.empty?
-
-            parameters.scan(PARAMETER_RE) do |name, unquoted_value, quoted_value|
-              # NOTE: Order of unquoted_value and quoted_value does not matter,
-              #       as they are mutually exclusive by the regex.
-              # @type var value: ::String?
-              value = unquoted_value || quoted_value
-              next if name.nil? || value.nil?
+                parameters[name] = value
+              end
+            end
 
-              # See https://github.com/soutaro/steep/issues/2051
-              name.downcase! # steep:ignore NoMethod
-              value.downcase!
+            new(type: type, subtype: subtype, parameters: parameters)
+          end
 
-              # See https://github.com/soutaro/steep/issues/2051
-              @parameters[name] = value # steep:ignore ArgumentTypeMismatch
-            end
+          def initialize(type:, subtype:, parameters: {})
+            @type = type
+            @subtype = subtype
+            @parameters = parameters
           end
 
           def to_s

data/lib/datadog/appsec/utils/http/url_encoded.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "cgi"
+
+module Datadog
+  module AppSec
+    module Utils
+      module HTTP
+        # Module for parsing URL encoded payloads
+        module URLEncoded
+          module_function
+
+          # Matches Rack's default query bytesize limit, so parsing our own
+          # payloads keeps the same guard against CPU/memory exhaustion.
+          DEFAULT_BYTESIZE_LIMIT = 4 * 1024 * 1024
+
+          AMPERSAND_BYTE = 0x26
+          EQUALS_BYTE = 0x3D
+
+          # Parses a URL encoded payload (query string or form data) into a hash
+          # of keys and values, merging duplicate keys.
+          #
+          # Example:
+          #
+          #   URLEncoded.parse("foo=bar&foo=baz&qux=quux") # => {"foo" => ["bar", "baz"], "qux" => "quux"}
+          #
+          # Parsing stops once +limit+ bytes have been read, and the pair being
+          # read at that point is discarded. This returns the pairs decoded so
+          # far rather than raising or discarding the whole payload.
+          def parse(payload, limit: DEFAULT_BYTESIZE_LIMIT)
+            raise ArgumentError, "limit must not be negative" if limit < 0
+
+            return {} if payload.nil? || payload.empty? || limit.zero?
+
+            # @type var result: URLEncoded::params
+            result = {}
+            payload_bytesize = payload.bytesize
+            bytes_to_parse = (payload_bytesize < limit) ? payload_bytesize : limit
+
+            index = 0
+            param_start = 0
+            equals_index = nil # : Integer?
+
+            payload.each_byte do |byte|
+              break if index >= bytes_to_parse
+
+              if byte == AMPERSAND_BYTE
+                param_end = index
+
+                if equals_index
+                  key = payload.byteslice(param_start, equals_index - param_start) # : String
+                  value = payload.byteslice(equals_index + 1, param_end - equals_index - 1) # : String?
+                else
+                  key = payload.byteslice(param_start, param_end - param_start) # : String
+                  value = nil
+                end
+
+                if !key.empty? || value
+                  key = CGI.unescape(key)
+                  value = CGI.unescape(value) if value
+
+                  add_param(result, key, value)
+                end
+
+                param_start = index + 1
+                equals_index = nil
+              elsif byte == EQUALS_BYTE
+                equals_index ||= index
+              end
+
+              index += 1
+            end
+
+            if bytes_to_parse == payload_bytesize && param_start < index
+              param_end = index
+
+              if equals_index
+                key = payload.byteslice(param_start, equals_index - param_start) # : String
+                value = payload.byteslice(equals_index + 1, param_end - equals_index - 1) # : String?
+              else
+                key = payload.byteslice(param_start, param_end - param_start) # : String
+                value = nil
+              end
+
+              if !key.empty? || value
+                key = CGI.unescape(key)
+                value = CGI.unescape(value) if value
+
+                add_param(result, key, value)
+              end
+            end
+
+            result
+          end
+
+          private_class_method def add_param(params, key, value)
+            case existing = params[key]
+            when ::Array then existing.push(value)
+            when nil then params[key] = value
+            else params[key] = [existing, value]
+            end
+          end
+        end
+      end
+    end
+  end
+end