api_hammer 0.8.1 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1fe246f792bb80940376f41dddb4f18d7578d0e
-  data.tar.gz: 22c1fd36b3196a1ec938fa0fc82c8f22daa2dece
+  metadata.gz: 25094fc3ff153feb9e2ac93c7c25a0e46d3143eb
+  data.tar.gz: 42cbb9d5226111853163fd94043cd33ee16eee70
 SHA512:
-  metadata.gz: 797705bc23ad885527c573a69e5b81d51f391e4d418aa1ce804f55e060660ba59c6e7824123fb6fa07d448c80720432e25346ae8d1410b4bb5047a7033025f48
-  data.tar.gz: 074ee7a9d90d371f8e6ed3edac7bd3fdc58c2cbd5a4c428e2f0a60ae5c7c48b385f786e5fec6422ce809937f82b10ab00be6385c98f643dec9627bef26cfd3c2
+  metadata.gz: f2b197a3c690c2e77a5cbebfcca7b6f98cbc1c4e5d7883a2cc1af545934be7433e58c5910eeffb77110e1da308a8753b993d5d2c683999192e18b178b4f26701
+  data.tar.gz: 351a18b0ac2e23726b7614743f7097402a17330d80a30a81fa2cb418791bd8a598d682eba94947ce9529d92d1bcdc02b5f4f25d99bc43a60ab2b769c177211d7

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+# v0.9.2
+- bugfix form encoded filtering
+
+# v0.9.1
+- recognize `app.config.api_hammer_request_logging_options` for request logger options
+
+# v0.9.0
+- rack request logger logs ids in arrays of hashes when logging ids
+- filtered logging of sensitive keys in bodies of requests (json and form encoded)
+- logstash filter for oauth headers and oauthenticator log entries
+
 # v0.8.1
 - request log format tweaks
 

data/lib/api_hammer/content_type_attrs.rb

@@ -0,0 +1,67 @@
+module ApiHammer
+  # parses attributes out of content type header
+  class ContentTypeAttrs
+    def initialize(content_type)
+      @media_type = content_type.split(/\s*[;]\s*/, 2).first if content_type
+      @media_type.strip! if @media_type
+      @content_type = content_type
+      @parsed = false
+      @attributes = Hash.new { |h,k| h[k] = [] }
+      catch(:unparseable) do
+        throw(:unparseable) unless content_type
+        uri_parser = URI.const_defined?(:Parser) ? URI::Parser.new : URI
+        scanner = StringScanner.new(content_type)
+        scanner.scan(/.*;\s*/) || throw(:unparseable)
+        while match = scanner.scan(/(\w+)=("?)([^"]*)("?)\s*(,?)\s*/)
+          key = scanner[1]
+          quote1 = scanner[2]
+          value = scanner[3]
+          quote2 = scanner[4]
+          comma_follows = !scanner[5].empty?
+          throw(:unparseable) unless quote1 == quote2
+          throw(:unparseable) if !comma_follows && !scanner.eos?
+          @attributes[uri_parser.unescape(key)] << uri_parser.unescape(value)
+        end
+        throw(:unparseable) unless scanner.eos?
+        @parsed = true
+      end
+    end
+
+    attr_reader :media_type
+
+    def parsed?
+      @parsed
+    end
+
+    def [](key)
+      @attributes[key]
+    end
+
+    def text?
+      # ordered hash by priority mapping types to binary or text
+      # regexps will have \A and \z added 
+      types = {
+        %r(image/.*) => :binary,
+        %r(audio/.*) => :binary,
+        %r(video/.*) => :binary,
+        %r(model/.*) => :binary,
+        %r(text/.*) => :text,
+        %r(message/.*) => :text,
+        'application/octet-stream' => :binary,
+        'application/ogg' => :binary,
+        'application/pdf' => :binary,
+        'application/postscript' => :binary,
+        'application/zip' => :binary,
+        'application/gzip' => :binary,
+      }
+      types.each do |match, type|
+        matched = match.is_a?(Regexp) ? media_type =~ %r(\A#{match.source}\z) : media_type == match
+        if matched
+          return type == :text
+        end
+      end
+      # fallback (unknown or not given) assume text
+      return true
+    end
+  end
+end

data/lib/api_hammer/faraday/request_logger.rb

@@ -1,57 +1,86 @@
 require 'faraday'
-require 'rack'
 require 'term/ansicolor'
 require 'json'
 require 'strscan'
 
 module ApiHammer
-  # parses attributes out of content type header
-  class ContentTypeAttrs
-    def initialize(content_type)
-      @media_type = content_type.split(/\s*[;]\s*/, 2).first if content_type
-      @media_type.strip! if @media_type
-      @content_type = content_type
-      @parsed = false
-      @attributes = Hash.new { |h,k| h[k] = [] }
-      catch(:unparseable) do
-        throw(:unparseable) unless content_type
-        uri_parser = URI.const_defined?(:Parser) ? URI::Parser.new : URI
-        scanner = StringScanner.new(content_type)
-        scanner.scan(/.*;\s*/) || throw(:unparseable)
-        while match = scanner.scan(/(\w+)=("?)([^"]*)("?)\s*(,?)\s*/)
-          key = scanner[1]
-          quote1 = scanner[2]
-          value = scanner[3]
-          quote2 = scanner[4]
-          comma_follows = !scanner[5].empty?
-          throw(:unparseable) unless quote1 == quote2
-          throw(:unparseable) if !comma_follows && !scanner.eos?
-          @attributes[uri_parser.unescape(key)] << uri_parser.unescape(value)
-        end
-        throw(:unparseable) unless scanner.eos?
-        @parsed = true
+  module Faraday
+    class Request
+      def initialize(request_env, response_env)
+        @request_env = request_env
+        @response_env = response_env
       end
-    end
 
-    attr_reader :media_type
+      attr_reader :request_env
+      attr_reader :response_env
 
-    def parsed?
-      @parsed
-    end
+      # deal with the vagaries of getting the response body in a form which JSON 
+      # gem will not cry about generating 
+      def response_body
+        instance_variable_defined?(:@response_body) ? @response_body : @response_body = catch(:response_body) do
+          unless response_env.body.is_a?(String)
+            begin
+              # if the response body is not a string, but JSON doesn't complain 
+              # about dumping whatever it is, go ahead and use it
+              JSON.generate([response_env.body])
+              throw :response_body, response_env.body
+            rescue
+              # otherwise return nil - don't know what to do with whatever this object is 
+              throw :response_body, nil
+            end
+          end
+
+          # first try to change the string's encoding per the Content-Type header 
+          content_type = response_env.response_headers['Content-Type']
+          response_body = response_env.body.dup
+          unless response_body.valid_encoding?
+            # I think this always comes in as ASCII-8BIT anyway so may never get here. hopefully.
+            response_body.force_encoding('ASCII-8BIT')
+          end
 
-    def [](key)
-      @attributes[key]
+          content_type_attrs = ContentTypeAttrs.new(content_type)
+          if content_type_attrs.parsed?
+            charset = content_type_attrs['charset'].first
+            if charset && Encoding.list.any? { |enc| enc.to_s.downcase == charset.downcase }
+              if response_body.dup.force_encoding(charset).valid_encoding?
+                response_body.force_encoding(charset)
+              else
+                # I guess just ignore the specified encoding if the result is not valid. fall back to 
+                # something else below.
+              end
+            end
+          end
+          begin
+            JSON.generate([response_body])
+          rescue Encoding::UndefinedConversionError
+            # if updating by content-type didn't do it, try UTF8 since JSON wants that - but only 
+            # if it seems to be valid utf8. 
+            # don't try utf8 if the response content-type indicated something else. 
+            try_utf8 = !(content_type_attrs && content_type_attrs.parsed? && content_type_attrs['charset'].any?)
+            if try_utf8 && response_body.dup.force_encoding('UTF-8').valid_encoding?
+              response_body.force_encoding('UTF-8')
+            else
+              # I'm not sure if there is a way in this situation to get JSON gem to generate the 
+              # string correctly. fall back to an array of codepoints I guess? this is a weird 
+              # solution but the best I've got for now. 
+              response_body = response_body.codepoints.to_a
+            end
+          end
+          response_body
+        end
+      end
     end
-  end
 
-  module Faraday
     # Faraday middleware for logging.
     #
-    # two lines:
+    # logs two lines:
     #
     # - an info line, colored prettily to show a brief summary of the request and response
     # - a debug line of json to record all relevant info. this is a lot of stuff jammed into one line, not 
     #   pretty, but informative.
+    #
+    # options:
+    # - :filter_keys defines keys whose values will be filtered out of the logging 
     class RequestLogger < ::Faraday::Middleware
       include Term::ANSIColor
 
@@ -61,89 +90,6 @@ module ApiHammer
         @options = options
       end
 
-      # deal with the vagaries of getting the response body in a form which JSON 
-      # gem will not cry about dumping 
-      def response_body(response_env)
-        unless response_env.body.is_a?(String)
-          begin
-            # if the response body is not a string, but JSON doesn't complain 
-            # about dumping whatever it is, go ahead and use it
-            JSON.dump([response_env.body])
-            return response_env.body
-          rescue
-            # otherwise return nil - don't know what to do with whatever this object is 
-            return nil
-          end
-        end
-
-        # first try to change the string's encoding per the Content-Type header 
-        content_type = response_env.response_headers['Content-Type']
-        response_body = response_env.body.dup
-        unless response_body.valid_encoding?
-          # I think this always comes in as ASCII-8BIT anyway so may never get here. hopefully.
-          response_body.force_encoding('ASCII-8BIT')
-        end
-
-        content_type_attrs = ContentTypeAttrs.new(content_type)
-        if content_type_attrs.parsed?
-          charset = content_type_attrs['charset'].first
-          if charset && Encoding.list.any? { |enc| enc.to_s.downcase == charset.downcase }
-            if response_body.dup.force_encoding(charset).valid_encoding?
-              response_body.force_encoding(charset)
-            else
-              # I guess just ignore the specified encoding if the result is not valid. fall back to 
-              # something else below.
-            end
-          end
-        end
-        begin
-          JSON.dump([response_body])
-        rescue Encoding::UndefinedConversionError
-          # if updating by content-type didn't do it, try UTF8 since JSON wants that - but only 
-          # if it seems to be valid utf8. 
-          # don't try utf8 if the response content-type indicated something else. 
-          try_utf8 = !(content_type_attrs && content_type_attrs.parsed? && content_type_attrs['charset'].any?)
-          if try_utf8 && response_body.dup.force_encoding('UTF-8').valid_encoding?
-            response_body.force_encoding('UTF-8')
-          else
-            # I'm not sure if there is a way in this situation to get JSON gem to dump the 
-            # string correctly. fall back to an array of codepoints I guess? this is a weird 
-            # solution but the best I've got for now. 
-            response_body = response_body.codepoints.to_a
-          end
-        end
-        response_body
-      end
-
-      def text?(content_type)
-        content_type_attrs = ContentTypeAttrs.new(content_type)
-        media_type = content_type_attrs.media_type
-        # ordered hash by priority mapping types to binary or text
-        # regexps will have \A and \z added 
-        types = {
-          %r(image/.*) => :binary,
-          %r(audio/.*) => :binary,
-          %r(video/.*) => :binary,
-          %r(model/.*) => :binary,
-          %r(text/.*) => :text,
-          %r(message/.*) => :text,
-          'application/octet-stream' => :binary,
-          'application/ogg' => :binary,
-          'application/pdf' => :binary,
-          'application/postscript' => :binary,
-          'application/zip' => :binary,
-          'application/gzip' => :binary,
-        }
-        types.each do |match, type|
-          matched = match.is_a?(Regexp) ? media_type =~ %r(\A#{match.source}\z) : media_type == match
-          if matched
-            return type == :text
-          end
-        end
-        # fallback (unknown or not given) assume text
-        return true
-      end
-
       def call(request_env)
         began_at = Time.now
 
@@ -155,6 +101,8 @@ module ApiHammer
         @app.call(request_env).on_complete do |response_env|
           now = Time.now
 
+          request = ApiHammer::Faraday::Request.new(request_env, response_env)
+
           status_color = case response_env.status.to_i
           when 200..299
             :intense_green
@@ -166,18 +114,32 @@ module ApiHammer
             :white
           end
           status_s = bold(send(status_color, response_env.status.to_s))
+
+          filtered_request_body = request_body.dup if request_body
+          filtered_response_body = request.response_body.dup if request.response_body
+
+          if @options[:filter_keys]
+            body_info = [['request', filtered_request_body, request_env.request_headers], ['response', filtered_response_body, response_env.response_headers]]
+            body_info.map do |(role, body, headers)|
+              if body
+                parsed_body = ApiHammer::ParsedBody.new(body, headers['Content-Type'])
+                body.replace(parsed_body.filtered_body(@options.slice(:filter_keys)))
+              end
+            end
+          end
+
           data = {
             'request_role' => 'client',
             'request' => {
               'method' => request_env[:method],
               'uri' => request_env[:url].normalize.to_s,
               'headers' => request_env.request_headers,
-              'body' => (request_body if text?(request_env.request_headers['Content-Type'])),
+              'body' => (filtered_request_body if ContentTypeAttrs.new(request_env.request_headers['Content-Type']).text?),
             }.reject{|k,v| v.nil? },
             'response' => {
               'status' => response_env.status.to_s,
               'headers' => response_env.response_headers,
-              'body' => (response_body(response_env) if text?(response_env.response_headers['Content-Type'])),
+              'body' => (filtered_response_body if ContentTypeAttrs.new(response_env.response_headers['Content-Type']).text?),
             }.reject{|k,v| v.nil? },
             'processing' => {
               'began_at' => began_at.utc.to_f,
@@ -186,7 +148,7 @@ module ApiHammer
             }.reject{|k,v| v.nil? },
           }
 
-          json_data = JSON.dump(data)
+          json_data = JSON.generate(data)
           dolog = proc do
             now_s = now.strftime('%Y-%m-%d %H:%M:%S %Z')
             @logger.info "#{bold(intense_magenta('>'))} #{status_s} : #{bold(intense_magenta(request_env[:method].to_s.upcase))} #{intense_magenta(request_env[:url].normalize.to_s)} @ #{intense_magenta(now_s)}"

data/lib/api_hammer/filtration/form_encoded.rb

@@ -0,0 +1,33 @@
+require 'strscan'
+
+module ApiHammer
+  module Filtration
+    class FormEncoded
+      def initialize(string, options = {})
+        @string = string
+        @options = options
+      end
+
+      def filter
+        ss = StringScanner.new(@string)
+        filtered = ''
+        while !ss.eos?
+          if ss.scan(/[&;]/)
+            filtered << ss[0]
+          end
+          if ss.scan(/[^&;]+/)
+            kv = ss[0]
+            key, value = kv.split('=', 2)
+            parsed_key = CGI::unescape(key)
+            if [*@options[:filter_keys]].any? { |fk| parsed_key =~ /(\A|[\[\]])#{Regexp.escape(fk)}(\z|[\[\]])/ }
+              filtered << [key, '[FILTERED]'].join('=')
+            else
+              filtered << ss[0]
+            end
+          end
+        end
+        filtered
+      end
+    end
+  end
+end

data/lib/api_hammer/filtration/json.rb

@@ -0,0 +1,137 @@
+require 'json'
+require 'json/pure/parser'
+
+module ApiHammer
+  module Filtration
+    class ParserError < JSON::ParserError; end
+
+    # This class implements the JSON filterer that is used to filter a JSON string
+    class Json < JSON::Pure::Parser
+      # Creates a new instance for the string _source_.
+      def initialize(source, opts = {})
+        super source
+        @options = opts
+      end
+
+      def filter
+        reset
+        obj = ''
+        while !eos? && scan_result(obj, IGNORE)
+        end
+        if eos?
+          raise ParserError, "source did not contain any JSON!"
+        else
+          value = filter_value
+          if value == UNPARSED
+            raise ParserError, "source did not contain any JSON!"
+          else
+            obj << value
+          end
+        end
+        obj
+      end
+
+      private
+
+      def filter_string
+        if scan(STRING)
+          self[0]
+        else
+          UNPARSED
+        end
+      end
+
+      def filter_value
+        simple = [FLOAT, INTEGER, TRUE, FALSE, NULL] + (@allow_nan ? [NAN, INFINITY, MINUS_INFINITY] : [])
+        if simple.any? { |type| scan(type) }
+          self[0]
+        elsif (string = filter_string) != UNPARSED
+          string
+        elsif scan(ARRAY_OPEN)
+          self[0] + filter_array
+        elsif scan(OBJECT_OPEN)
+          self[0] + filter_object
+        else
+          UNPARSED
+        end
+      end
+
+      def filter_array
+        result = ''
+        delim = false
+        until eos?
+          if (value = filter_value) != UNPARSED
+            delim = false
+            result << value
+            scan_result(result, IGNORE)
+            if scan_result(result, COLLECTION_DELIMITER)
+              delim = true
+            elsif !match?(ARRAY_CLOSE)
+              raise ParserError, "expected ',' or ']' in array at '#{peek(20)}'!"
+            end
+          elsif scan_result(result, ARRAY_CLOSE)
+            if delim
+              raise ParserError, "expected next element in array at '#{peek(20)}'!"
+            end
+            break
+          elsif scan_result(result, IGNORE)
+            #
+          else
+            raise ParserError, "unexpected token in array at '#{peek(20)}'!"
+          end
+        end
+        result
+      end
+
+      FILTERED_JSON = JSON.generate("[FILTERED]", :quirks_mode => true)
+
+      def filter_object
+        result = ''
+        delim = false
+        until eos?
+          if (string = filter_string) != UNPARSED
+            parsed_key = JSON.parse(string, :quirks_mode => true)
+            result << string
+            scan_result(result, IGNORE)
+            unless scan_result(result, PAIR_DELIMITER)
+              raise ParserError, "expected ':' in object at '#{peek(20)}'!"
+            end
+            scan_result(result, IGNORE)
+            unless (value = filter_value).equal? UNPARSED
+              if [*@options[:filter_keys]].include?(parsed_key)
+                result << FILTERED_JSON
+              else
+                result << value
+              end
+              delim = false
+              scan_result(result, IGNORE)
+              if scan_result(result, COLLECTION_DELIMITER)
+                delim = true
+              elsif !match?(OBJECT_CLOSE)
+                raise ParserError, "expected ',' or '}' in object at '#{peek(20)}'!"
+              end
+            else
+              raise ParserError, "expected value in object at '#{peek(20)}'!"
+            end
+          elsif scan_result(result, OBJECT_CLOSE)
+            if delim
+              raise ParserError, "expected next name, value pair in object at '#{peek(20)}'!"
+            end
+            break
+          elsif scan_result(result, IGNORE)
+            #
+          else
+            raise ParserError, "unexpected token in object at '#{peek(20)}'!"
+          end
+        end
+        result
+      end
+
+      def scan_result(result, match)
+        if scan(match)
+          result << self[0]
+        end
+      end
+    end
+  end
+end

data/lib/api_hammer/parsed_body.rb

@@ -0,0 +1,39 @@
+require 'rack'
+
+module ApiHammer
+  class ParsedBody
+    attr_reader :body, :content_type, :media_type
+
+    def initialize(body, content_type)
+      @body = body
+      @content_type = content_type
+      @media_type = ::Rack::Request.new({'CONTENT_TYPE' => content_type}).media_type
+    end
+
+    def object
+      instance_variable_defined?(:@object) ? @object : @object = begin
+        if media_type == 'application/json'
+          JSON.parse(body) rescue nil
+        elsif media_type == 'application/x-www-form-urlencoded'
+          CGI.parse(body).map { |k, vs| {k => vs.last} }.inject({}, &:update)
+        end
+      end
+    end
+
+    def filtered_body(options)
+      @filtered_body ||= begin
+        if media_type == 'application/json'
+          begin
+            ApiHammer::Filtration::Json.new(body, options).filter
+          rescue JSON::ParserError
+            body
+          end
+        elsif media_type == 'application/x-www-form-urlencoded'
+          ApiHammer::Filtration::FormEncoded.new(body, options).filter
+        else
+          body
+        end
+      end
+    end
+  end
+end

data/lib/api_hammer/rails_request_logging.rb

@@ -51,7 +51,8 @@ module ApiHammer
 
       ApiHammer::RequestLogSubscriber.attach_to :action_controller
 
-      app.config.middleware.insert_after(::Rails::Rack::Logger, ApiHammer::RequestLogger, ::Rails.logger)
+      options = app.config.respond_to?(:api_hammer_request_logging_options) ? app.config.api_hammer_request_logging_options : {}
+      app.config.middleware.insert_after(::Rails::Rack::Logger, ApiHammer::RequestLogger, ::Rails.logger, options)
 
       ActionController::Base.send(:include, AddRequestToPayload)
     end

data/lib/api_hammer/request_logger.rb

@@ -17,6 +17,14 @@ module ApiHammer
 
     LARGE_BODY_SIZE = 4096
 
+    # options
+    # - :logger
+    # - :filter_keys
+    def initialize(app, logger, options={})
+      @options = options
+      super(app, logger)
+    end
+
     def call(env)
       began_at = Time.now
 
@@ -102,22 +110,21 @@ module ApiHammer
       response_body_string = response_body.to_enum.to_a.join('')
       body_info = [['request', request_body, request.content_type], ['response', response_body_string, response.content_type]]
       body_info.map do |(role, body, content_type)|
+        parsed_body = ApiHammer::ParsedBody.new(body, content_type)
         if (400..599).include?(status.to_i) || body.size < LARGE_BODY_SIZE
           # log bodies if they are not large, or if there was an error (either client or server) 
-          data[role]['body'] = body
+          data[role]['body'] = parsed_body.filtered_body(@options.reject { |k,v| ![:filter_keys].include?(k) })
         else
           # otherwise, log id and uuid fields 
-          media_type = ::Rack::Request.new({'CONTENT_TYPE' => content_type}).media_type
-          body_object = begin
-            if media_type == 'application/json'
-              JSON.parse(body) rescue nil
-            elsif media_type == 'application/x-www-form-urlencoded'
-              CGI.parse(body).map { |k, vs| {k => vs.last} }.inject({}, &:update)
-            end
+          body_object = parsed_body.object
+          sep = /(?:\b|\W|_)/
+          hash_ids = proc do |hash|
+            hash.reject { |key, value| !(key =~ /#{sep}([ug]u)?id#{sep}/ && value.is_a?(String)) }
           end
           if body_object.is_a?(Hash)
-            sep = /(?:\b|\W|_)/
-            body_ids = body_object.reject { |key, value| !(key =~ /#{sep}([ug]u)?id#{sep}/ && value.is_a?(String)) }
+            body_ids = hash_ids.call(body_object)
+          elsif body_object.is_a?(Array) && body_object.all? { |e| e.is_a?(Hash) }
+            body_ids = body_object.map(&hash_ids)
           end
 
           data[role]['body_ids'] = body_ids if body_ids && body_ids.any?

data/lib/api_hammer/version.rb

@@ -1,3 +1,3 @@
 module ApiHammer
-  VERSION = "0.8.1"
+  VERSION = "0.9.2"
 end

data/lib/api_hammer.rb

@@ -10,9 +10,15 @@ module ApiHammer
   autoload :RailsOrSidekiqLogger, 'api_hammer/rails_or_sidekiq_logger'
   autoload :FaradayOutputter, 'api_hammer/faraday/outputter'
   autoload :FaradayCurlVOutputter, 'api_hammer/faraday/outputter'
+  autoload :ParsedBody, 'api_hammer/parsed_body'
+  autoload :ContentTypeAttrs, 'api_hammer/content_type_attrs'
   module Faraday
     autoload :RequestLogger, 'api_hammer/faraday/request_logger'
   end
+  module Filtration
+    autoload :Json, 'api_hammer/filtration/json'
+    autoload :FormEncoded, 'api_hammer/filtration/form_encoded'
+  end
 end
 
 require 'faraday'

data/lib/logstash/filters/api_hammer_request.rb

@@ -16,6 +16,7 @@ class LogStash::Filters::ApiHammerRequest < LogStash::Filters::Base
   def filter(event)
     # discard the request status line for humans - always followed by json which we'll parse 
     col = /[\e\[\dm]*/.source
+    #               begin  direction     status        method      path             time      end
     human_request = [/\A/, /[<>]/, /\s/, /\d+/, / : /, /\w+/, / /, /[^\e]+/, / @ /, /[^\e]+/, /\z/].map(&:source).join(col)
     event.cancel if event[@source] =~ /#{human_request}/
 

data/lib/logstash/filters/oauthenticator.rb

@@ -0,0 +1,45 @@
+require "logstash/filters/base"
+require "logstash/namespace"
+require 'oauthenticator'
+
+class LogStash::Filters::OAuthenticator < LogStash::Filters::Base
+  config_name "oauthenticator"
+  milestone 1
+
+  config :consume, :validate => :boolean, :default => true
+  config :source, :validate => :string, :default => 'message'
+
+  public
+  def register
+  end
+
+  public
+  def filter(event)
+    #OAuthenticator authenticated an authentic request with Authorization: OAuth realm="", oauth_consumer_key="ios-production-lFo4Zqgs", oauth_token="aE7wU1VPPa7G2l2VLtVRalgIOM4zKJUu7BMnQZoH", oauth_signature_method="HMAC-SHA1", oauth_version="1.0", oauth_nonce="34DA75CB-7653-4AF5-A3F8-B0989AABFCDF", oauth_timestamp="1411935761", oauth_signature="H%2F0kt3aSPqdo2qgfRrbPPirR%2F4g%3D"
+    match = event[@source].match(/\A(OAuthenticator authenticated an authentic request) with Authorization: /)
+    if match
+      authorization = match.post_match
+
+      begin
+        event['oauth'] = OAuthenticator.parse_authorization(authorization)
+      rescue OAuthenticator::Error => parse_exception
+        nil
+      end
+
+      event[@source] = match[1] if @consume
+    end
+
+    # parse the authorization of a request filtered by LogStash::Filters::ApiHammerRequest
+    if event['request'].is_a?(Hash) && event['request']['headers'].is_a?(Hash)
+      authorization = event['request']['headers'].inject(nil) { |a, (k,v)| k.is_a?(String) && k.downcase == 'authorization' ? v : a }
+      if authorization.is_a?(String)
+        begin
+          event['request']['oauth'] = OAuthenticator.parse_authorization(authorization)
+        rescue OAuthenticator::Error => parse_exception
+          # if it is not oauth or badly formed oauth we don't care 
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/logstash/filters/request_bodies_parsed.rb

@@ -3,6 +3,8 @@ require "logstash/namespace"
 require 'rack'
 require 'cgi'
 require 'json'
+require 'api_hammer/parsed_body'
+
 class LogStash::Filters::RequestBodiesParsed < LogStash::Filters::Base
   config_name "request_bodies_parsed"
   milestone 1
@@ -19,15 +21,8 @@ class LogStash::Filters::RequestBodiesParsed < LogStash::Filters::Base
         if event[re]['headers'].is_a?(Hash) && !content_type
           _, content_type = event[re]['headers'].detect { |(k,_)| k =~ /\Acontent.type\z/i }
         end
-        media_type = ::Rack::Request.new({'CONTENT_TYPE' => content_type}).media_type
-        body_parsed = begin
-          if media_type == 'application/json'
-            JSON.parse(event[re]['body']) rescue nil
-          elsif media_type == 'application/x-www-form-urlencoded'
-            CGI.parse(event[re]['body']).map { |k, vs| {k => vs.last} }.inject({}, &:update)
-          end
-        end
-        event[re]['body_parsed'] = body_parsed if body_parsed
+        parsed_body = ApiHammer::ParsedBody.new(event[re]['body'], content_type)
+        event[re]['body_parsed'] = parsed_body.object if parsed_body.object
       end
     end
   end

data/test/faraday_request_logger_test.rb

@@ -103,7 +103,8 @@ describe ApiHammer::RequestLogger do
       conn.get '/'
       assert_match('[120,120,195]', logio.string)
     end
-
+  end
+  describe 'logging body by content-type' do
     {
       'application/octet-stream' => false,
       'image/png' => false,
@@ -124,4 +125,90 @@ describe ApiHammer::RequestLogger do
       end
     end
   end
+
+  describe 'filtering' do
+    describe 'json response' do
+      it 'filters' do
+        app = proc { |env| [200, {'Content-Type' => 'application/json'}, ['{"pin": "foobar", "bar": "baz"}']] }
+        conn = Faraday.new do |f|
+          f.request :api_hammer_request_logger, logger, :filter_keys => 'pin'
+          f.use Faraday::Adapter::Rack, app
+        end
+        conn.get '/'
+        assert_includes(logio.string, %q("body":"{\"pin\": \"[FILTERED]\", \"bar\": \"baz\"}"))
+      end
+      it 'filters nested' do
+        app = proc { |env| [200, {'Content-Type' => 'application/json'}, ['{"object": {"pin": "foobar"}, "bar": "baz"}']] }
+        conn = Faraday.new do |f|
+          f.request :api_hammer_request_logger, logger, :filter_keys => 'pin'
+          f.use Faraday::Adapter::Rack, app
+        end
+        conn.get '/'
+        assert_includes(logio.string, %q("body":"{\"object\": {\"pin\": \"[FILTERED]\"}, \"bar\": \"baz\"}"))
+      end
+      it 'filters in array' do
+        app = proc { |env| [200, {'Content-Type' => 'application/json'}, ['[{"object": [{"pin": ["foobar"]}], "bar": "baz"}]']] }
+        conn = Faraday.new do |f|
+          f.request :api_hammer_request_logger, logger, :filter_keys => 'pin'
+          f.use Faraday::Adapter::Rack, app
+        end
+        conn.get '/'
+        assert_includes(logio.string, %q("body":"[{\"object\": [{\"pin\": \"[FILTERED]\"}], \"bar\": \"baz\"}]"))
+      end
+    end
+
+    describe 'json request' do
+      it 'filters a json request' do
+        app = proc { |env| [200, {}, []] }
+        conn = Faraday.new do |f|
+          f.request :api_hammer_request_logger, logger, :filter_keys => 'pin'
+          f.use Faraday::Adapter::Rack, app
+        end
+        conn.post '/', '[{"object": [{"pin": ["foobar"]}], "bar": "baz"}]', {'Content-Type' => 'application/json'}
+        assert_includes(logio.string, %q("body":"[{\"object\": [{\"pin\": \"[FILTERED]\"}], \"bar\": \"baz\"}]"))
+      end
+    end
+
+    describe 'form encoded response' do
+      it 'filters' do
+        app = proc { |env| [200, {'Content-Type' => 'application/x-www-form-urlencoded'}, ['pin=foobar&bar=baz']] }
+        conn = Faraday.new do |f|
+          f.request :api_hammer_request_logger, logger, :filter_keys => 'pin'
+          f.use Faraday::Adapter::Rack, app
+        end
+        conn.get '/'
+        assert_includes(logio.string, %q("body":"pin=[FILTERED]&bar=baz"))
+      end
+      it 'filters nested' do
+        app = proc { |env| [200, {'Content-Type' => 'application/x-www-form-urlencoded'}, ['object[pin]=foobar&bar=baz']] }
+        conn = Faraday.new do |f|
+          f.request :api_hammer_request_logger, logger, :filter_keys => 'pin'
+          f.use Faraday::Adapter::Rack, app
+        end
+        conn.get '/'
+        assert_includes(logio.string, %q("body":"object[pin]=[FILTERED]&bar=baz"))
+      end
+      it 'filters in array' do
+        app = proc { |env| [200, {'Content-Type' => 'application/x-www-form-urlencoded'}, ['object[][pin][]=foobar&bar=baz']] }
+        conn = Faraday.new do |f|
+          f.request :api_hammer_request_logger, logger, :filter_keys => 'pin'
+          f.use Faraday::Adapter::Rack, app
+        end
+        conn.get '/'
+        assert_includes(logio.string, %q("body":"object[][pin][]=[FILTERED]&bar=baz"))
+      end
+    end
+
+    describe 'form encoded request' do
+      it 'filters a json request' do
+        app = proc { |env| [200, {}, []] }
+        conn = Faraday.new do |f|
+          f.request :api_hammer_request_logger, logger, :filter_keys => 'pin'
+          f.use Faraday::Adapter::Rack, app
+        end
+        conn.post '/', 'object[pin]=foobar&bar=baz', {'Content-Type' => 'application/x-www-form-urlencoded'}
+        assert_includes(logio.string, %q(object[pin]=[FILTERED]&bar=baz))
+      end
+    end
+  end
 end

data/test/request_logger_test.rb

@@ -28,6 +28,13 @@ describe ApiHammer::RequestLogger do
     assert_match(%q("body_ids":{"uuid":"theuuid","foo_uuid":"thefoouuid","id":"theid","id_for_x":"theidforx","bar.id":"thebarid","baz-guid":"bazzz"}), logio.string)
   end
 
+  it 'logs id and uuid (json array)' do
+    body = %Q([{"uuid": "theuuid", "foo_uuid": "thefoouuid"}, {"id": "theid", "id_for_x": "theidforx"}, {"bar.id": "thebarid", "baz-guid": "bazzz", "bigthing": "#{' ' * 4096}"}])
+    app = ApiHammer::RequestLogger.new(proc { |env| [200, {"Content-Type" => 'application/json; charset=UTF8'}, [body]] }, logger)
+    app.call(Rack::MockRequest.env_for('/')).last.close
+    assert_match(%q("body_ids":[{"uuid":"theuuid","foo_uuid":"thefoouuid"},{"id":"theid","id_for_x":"theidforx"},{"bar.id":"thebarid","baz-guid":"bazzz"}]), logio.string)
+  end
+
   it 'logs id and uuid (form encoded)' do
     body = %Q(uuid=theuuid&foo_uuid=thefoouuid&id=theid&id_for_x=theidforx&bar.id=thebarid&baz-guid=bazzz&bigthing=#{' ' * 4096})
     app = ApiHammer::RequestLogger.new(proc { |env| [200, {"Content-Type" => 'application/x-www-form-urlencoded; charset=UTF8'}, [body]] }, logger)
@@ -48,4 +55,71 @@ describe ApiHammer::RequestLogger do
     assert_match(/"request":\{.*"body":"the_request_body/, logio.string)
     assert_match(/"response":\{.*"body":"the_response_body/, logio.string)
   end
+
+  describe 'filtering' do
+    describe 'json response' do
+      it 'filters' do
+        body = %Q({"pin": "foobar"})
+        app = proc { |env| [200, {"Content-Type" => 'application/json; charset=UTF8'}, [body]] }
+        app = ApiHammer::RequestLogger.new(app, logger, :filter_keys => 'pin')
+        app.call(Rack::MockRequest.env_for('/')).last.close
+
+        assert_includes(logio.string, %q("body":"{\"pin\": \"[FILTERED]\"}"))
+      end
+      it 'filters nested' do
+        body = %Q({"object": {"pin": "foobar"}})
+        app = proc { |env| [200, {"Content-Type" => 'application/json; charset=UTF8'}, [body]] }
+        app = ApiHammer::RequestLogger.new(app, logger, :filter_keys => 'pin')
+        app.call(Rack::MockRequest.env_for('/')).last.close
+
+        assert_includes(logio.string, %q("body":"{\"object\": {\"pin\": \"[FILTERED]\"}}"))
+      end
+      it 'filters in array' do
+        body = %Q([{"object": [{"pin": ["foobar"]}]}])
+        app = proc { |env| [200, {"Content-Type" => 'application/json; charset=UTF8'}, [body]] }
+        app = ApiHammer::RequestLogger.new(app, logger, :filter_keys => 'pin')
+        app.call(Rack::MockRequest.env_for('/')).last.close
+
+        assert_includes(logio.string, %q("body":"[{\"object\": [{\"pin\": \"[FILTERED]\"}]}]"))
+      end
+    end
+
+    describe 'json request' do
+      it 'filters a json request' do
+        app = ApiHammer::RequestLogger.new(proc { |env| [200, {}, []] }, logger, :filter_keys => 'pin')
+        app.call(Rack::MockRequest.env_for('/', :input => '[{"object": [{"pin": ["foobar"]}]}]', 'CONTENT_TYPE' => 'application/json')).last.close
+        assert_includes(logio.string, %q("body":"[{\"object\": [{\"pin\": \"[FILTERED]\"}]}]"))
+      end
+    end
+
+    describe('form encoded response') do
+      it 'filters' do
+        app = proc { |env| [200, {"Content-Type" => 'application/x-www-form-urlencoded'}, ['pin=foobar']] }
+        app = ApiHammer::RequestLogger.new(app, logger, :filter_keys => 'pin')
+        app.call(Rack::MockRequest.env_for('/')).last.close
+        assert_includes(logio.string, %q("body":"pin=[FILTERED]"))
+      end
+      it 'filters nested' do
+        app = proc { |env| [200, {'Content-Type' => 'application/x-www-form-urlencoded'}, ['object[pin]=foobar']] }
+        app = ApiHammer::RequestLogger.new(app, logger, :filter_keys => 'pin')
+        app.call(Rack::MockRequest.env_for('/')).last.close
+        assert_includes(logio.string, %q("body":"object[pin]=[FILTERED]"))
+      end
+      it 'filters in array' do
+        app = proc { |env| [200, {'Content-Type' => 'application/x-www-form-urlencoded'}, ['object[][pin][]=foobar']] }
+        app = ApiHammer::RequestLogger.new(app, logger, :filter_keys => 'pin')
+        app.call(Rack::MockRequest.env_for('/')).last.close
+        assert_includes(logio.string, %q("body":"object[][pin][]=[FILTERED]"))
+      end
+    end
+
+    describe 'form encoded request' do
+      it 'filters a json request' do
+        app = proc { |env| [200, {}, []] }
+        app = ApiHammer::RequestLogger.new(app, logger, :filter_keys => 'pin')
+        app.call(Rack::MockRequest.env_for('/', :input => 'object[pin]=foobar', 'CONTENT_TYPE' => 'application/x-www-form-urlencoded')).last.close
+        assert_includes(logio.string, %q(object[pin]=[FILTERED]))
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: api_hammer
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.2
 platform: ruby
 authors:
 - Ethan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-19 00:00:00.000000000 Z
+date: 2015-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -239,9 +239,13 @@ files:
 - lib/api_hammer.rb
 - lib/api_hammer/active_record_cache_find_by.rb
 - lib/api_hammer/check_required_params.rb
+- lib/api_hammer/content_type_attrs.rb
 - lib/api_hammer/faraday/outputter.rb
 - lib/api_hammer/faraday/request_logger.rb
+- lib/api_hammer/filtration/form_encoded.rb
+- lib/api_hammer/filtration/json.rb
 - lib/api_hammer/halt.rb
+- lib/api_hammer/parsed_body.rb
 - lib/api_hammer/public_instance_exec.rb
 - lib/api_hammer/rails.rb
 - lib/api_hammer/rails_or_sidekiq_logger.rb
@@ -257,6 +261,7 @@ files:
 - lib/api_hammer/weblink.rb
 - lib/logstash/filters/active_support_tags.rb
 - lib/logstash/filters/api_hammer_request.rb
+- lib/logstash/filters/oauthenticator.rb
 - lib/logstash/filters/request_bodies_parsed.rb
 - lib/logstash/filters/ruby_logger.rb
 - lib/logstash/filters/sidekiq.rb