api_hammer 0.4.3 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2249d364c2e8a31b69f9e98e284ec2160b2a58ef
-  data.tar.gz: 1feb8ef43dc7d861f4aa581924a6e217b074abf4
+  metadata.gz: b342d8c401e21b4f1de5a9bca122654c97029575
+  data.tar.gz: 43ebdf3104bb5f99a077b1f5a6865175be99ae0b
 SHA512:
-  metadata.gz: 1842347f2fab562854fb809a7f73dd601cea6cca5cf02ab2f4cc02e2440eb168ac1d85fe65478b9c01af8f6b6f6acfb162ba5d105afcaf779bcd5d5f18c56bf7
-  data.tar.gz: 1affa946256f016d0e86ada2948f85474e94b93bf8e74944abc71b246aaac88b33118dce62dbf588b1b179d5da42802d5021590bab3b93397cb3684019e03dc0
+  metadata.gz: 354e49d66f033993d760bb2e465e8166330ec28be9f2bc7cf2d1c385fb887b3c5a06e744f60be19cbb3ed52c84024c41d6603004860628b50ecc37b6f3b94b88
+  data.tar.gz: 38e03dedd7c08c876b1f91cb7fd3b68da53b3dc045e71787d753a653d0952c10d884a6f36d7ca993e9de533b2fc19cd5462daba5131c4847250b2fb2324cfb6c

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# v0.5.0
+- rack request logger logs all request and response headers
+- fix id / uuid / guid logging in rack request logger
+- faraday request logger does not log binary bodies 
+
 # 0.4.3
 - bugfix
 

data/lib/api_hammer/faraday/request_logger.rb

@@ -12,6 +12,45 @@ if Faraday::Request.respond_to?(:register_middleware)
 end
 
 module ApiHammer
+  # parses attributes out of content type header
+  class ContentTypeAttrs
+    def initialize(content_type)
+      @media_type = content_type.split(/\s*[;]\s*/, 2).first if content_type
+      @media_type.strip! if @media_type
+      @content_type = content_type
+      @parsed = false
+      @attributes = Hash.new { |h,k| h[k] = [] }
+      catch(:unparseable) do
+        throw(:unparseable) unless content_type
+        uri_parser = URI.const_defined?(:Parser) ? URI::Parser.new : URI
+        scanner = StringScanner.new(content_type)
+        scanner.scan(/.*;\s*/) || throw(:unparseable)
+        while match = scanner.scan(/(\w+)=("?)([^"]*)("?)\s*(,?)\s*/)
+          key = scanner[1]
+          quote1 = scanner[2]
+          value = scanner[3]
+          quote2 = scanner[4]
+          comma_follows = !scanner[5].empty?
+          throw(:unparseable) unless quote1 == quote2
+          throw(:unparseable) if !comma_follows && !scanner.eos?
+          @attributes[uri_parser.unescape(key)] << uri_parser.unescape(value)
+        end
+        throw(:unparseable) unless scanner.eos?
+        @parsed = true
+      end
+    end
+
+    attr_reader :media_type
+
+    def parsed?
+      @parsed
+    end
+
+    def [](key)
+      @attributes[key]
+    end
+  end
+
   class Faraday
     # Faraday middleware for logging.
     #
@@ -40,36 +79,15 @@ module ApiHammer
           response_body.force_encoding('ASCII-8BIT')
         end
 
-        if content_type
-          # TODO refactor this parsing somewhere better? 
-          parsed = false
-          attributes = Hash.new { |h,k| h[k] = [] }
-          catch(:unparseable) do
-            uri_parser = URI.const_defined?(:Parser) ? URI::Parser.new : URI
-            scanner = StringScanner.new(content_type)
-            scanner.scan(/.*;\s*/) || throw(:unparseable)
-            while match = scanner.scan(/(\w+)=("?)([^"]*)("?)\s*(,?)\s*/)
-              key = scanner[1]
-              quote1 = scanner[2]
-              value = scanner[3]
-              quote2 = scanner[4]
-              comma_follows = !scanner[5].empty?
-              throw(:unparseable) unless quote1 == quote2
-              throw(:unparseable) if !comma_follows && !scanner.eos?
-              attributes[uri_parser.unescape(key)] << uri_parser.unescape(value)
-            end
-            throw(:unparseable) unless scanner.eos?
-            parsed = true
-          end
-          if parsed
-            charset = attributes['charset'].first
-            if charset && Encoding.list.any? { |enc| enc.to_s.downcase == charset.downcase }
-              if response_body.dup.force_encoding(charset).valid_encoding?
-                response_body.force_encoding(charset)
-              else
-                # I guess just ignore the specified encoding if the result is not valid. fall back to 
-                # something else below.
-              end
+        content_type_attrs = ContentTypeAttrs.new(content_type)
+        if content_type_attrs.parsed?
+          charset = content_type_attrs['charset'].first
+          if charset && Encoding.list.any? { |enc| enc.to_s.downcase == charset.downcase }
+            if response_body.dup.force_encoding(charset).valid_encoding?
+              response_body.force_encoding(charset)
+            else
+              # I guess just ignore the specified encoding if the result is not valid. fall back to 
+              # something else below.
             end
           end
         end
@@ -79,7 +97,7 @@ module ApiHammer
           # if updating by content-type didn't do it, try UTF8 since JSON wants that - but only 
           # if it seems to be valid utf8. 
           # don't try utf8 if the response content-type indicated something else. 
-          try_utf8 = !(parsed && attributes['charset'].any?)
+          try_utf8 = !(content_type_attrs && content_type_attrs.parsed? && content_type_attrs['charset'].any?)
           if try_utf8 && response_body.dup.force_encoding('UTF-8').valid_encoding?
             response_body.force_encoding('UTF-8')
           else
@@ -92,6 +110,35 @@ module ApiHammer
         response_body
       end
 
+      def text?(content_type)
+        content_type_attrs = ContentTypeAttrs.new(content_type)
+        media_type = content_type_attrs.media_type
+        # ordered hash by priority mapping types to binary or text
+        # regexps will have \A and \z added 
+        types = {
+          %r(image/.*) => :binary,
+          %r(audio/.*) => :binary,
+          %r(video/.*) => :binary,
+          %r(model/.*) => :binary,
+          %r(text/.*) => :text,
+          %r(message/.*) => :text,
+          'application/octet-stream' => :binary,
+          'application/ogg' => :binary,
+          'application/pdf' => :binary,
+          'application/postscript' => :binary,
+          'application/zip' => :binary,
+          'application/gzip' => :binary,
+        }
+        types.each do |match, type|
+          matched = match.is_a?(Regexp) ? media_type =~ %r(\A#{match.source}\z) : media_type == match
+          if matched
+            return type == :text
+          end
+        end
+        # fallback (unknown or not given) assume text
+        return true
+      end
+
       def call(request_env)
         began_at = Time.now
 
@@ -119,12 +166,12 @@ module ApiHammer
               'method' => request_env[:method],
               'uri' => request_env[:url].normalize.to_s,
               'headers' => request_env.request_headers,
-              'body' => request_body,
+              'body' => (request_body if text?(request_env.request_headers['Content-Type'])),
             }.reject{|k,v| v.nil? },
             'response' => {
               'status' => response_env.status,
               'headers' => response_env.response_headers,
-              'body' => response_body(response_env),
+              'body' => (response_body(response_env) if text?(response_env.response_headers['Content-Type'])),
             }.reject{|k,v| v.nil? },
             'processing' => {
               'began_at' => began_at.utc.to_i,

data/lib/api_hammer/request_logger.rb

@@ -20,23 +20,25 @@ module ApiHammer
 
       # this is closed after the app is called, so read it before 
       env["rack.input"].rewind
-      @request_body = env["rack.input"].read
+      request_body = env["rack.input"].read
       env["rack.input"].rewind
 
       log_tags = Thread.current[:activesupport_tagged_logging_tags]
-      @log_tags = log_tags.dup if log_tags && log_tags.any?
+      log_tags = log_tags.dup if log_tags && log_tags.any?
 
-      status, headers, body = @app.call(env)
-      headers = ::Rack::Utils::HeaderHash.new(headers)
-      body_proxy = ::Rack::BodyProxy.new(body) { log(env, status, headers, began_at, body) }
-      [status, headers, body_proxy]
+      status, response_headers, response_body = @app.call(env)
+      response_headers = ::Rack::Utils::HeaderHash.new(response_headers)
+      body_proxy = ::Rack::BodyProxy.new(response_body) do
+        log(env, request_body, status, response_headers, response_body, began_at, log_tags)
+      end
+      [status, response_headers, body_proxy]
     end
 
-    def log(env, status, headers, began_at, body)
+    def log(env, request_body, status, response_headers, response_body, began_at, log_tags)
       now = Time.now
 
       request = Rack::Request.new(env)
-      response = Rack::Response.new('', status, headers)
+      response = Rack::Response.new('', status, response_headers)
 
       request_uri = Addressable::URI.new(
         :scheme => request.scheme,
@@ -56,51 +58,67 @@ module ApiHammer
         :white
       end
       status_s = bold(send(status_color, status.to_s))
+
+      request_headers = env.map do |(key, value)|
+        http_match = key.match(/\AHTTP_/)
+        if http_match
+          name = http_match.post_match.downcase
+          {name => value}
+        else
+          name = %w(content_type content_length).detect { |name| name.downcase == key.downcase }
+          if name
+            {name => value}
+          end
+        end
+      end.compact.inject({}, &:update)
+
       data = {
         'request' => {
           'method' => request.request_method,
           'uri' => request_uri.normalize.to_s,
-          'length' => request.content_length,
-          'Content-Type' => request.content_type,
+          'headers' => request_headers,
           'remote_addr' => env['HTTP_X_FORWARDED_FOR'] || env["REMOTE_ADDR"],
-          'User-Agent' => request.user_agent,
           # these come from the OAuthenticator gem/middleware 
           'oauth.authenticated' => env['oauth.authenticated'],
           'oauth.consumer_key' => env['oauth.consumer_key'],
           'oauth.token' => env['oauth.token'],
           # airbrake
           'airbrake.error_id' => env['airbrake.error_id'],
-        }.reject{|k,v| v.nil? },
+        }.reject { |k,v| v.nil? },
         'response' => {
           'status' => status,
-          'length' => headers['Content-Length'] || body.to_enum.map(&::Rack::Utils.method(:bytesize)).inject(0, &:+),
-          'Location' => response.location,
-          'Content-Type' => response.content_type,
-        }.reject{|k,v| v.nil? },
+          'headers' => response_headers,
+          'length' => response_headers['Content-Length'] || response_body.to_enum.map(&::Rack::Utils.method(:bytesize)).inject(0, &:+),
+        }.reject { |k,v| v.nil? },
         'processing' => {
           'began_at' => began_at.utc.to_i,
           'duration' => now - began_at,
-          'activesupport_tagged_logging_tags' => @log_tags,
-        }.merge(env['request_logger.info'] || {}).merge(Thread.current['request_logger.info'] || {}).reject{|k,v| v.nil? },
+          'activesupport_tagged_logging_tags' => log_tags,
+        }.merge(env['request_logger.info'] || {}).merge(Thread.current['request_logger.info'] || {}).reject { |k,v| v.nil? },
       }
       ids_from_body = proc do |body_string, content_type|
         media_type = ::Rack::Request.new({'CONTENT_TYPE' => content_type}).media_type
-        if media_type == 'application/json'
-          body_object = JSON.parse(body_string) rescue nil
-        # TODO form encoded
+        body_object = begin
+          if media_type == 'application/json'
+            JSON.parse(body_string) rescue nil
+          elsif media_type == 'application/x-www-form-urlencoded'
+            CGI.parse(body_string).map { |k, vs| {k => vs.last} }.inject({}, &:update)
+          end
         end
         if body_object.is_a?(Hash)
-          body_object.reject { |key, value| !(key =~ /\b(uu)?id\b/ && value.is_a?(String)) }
+          sep = /(?:\b|\W|_)/
+          body_object.reject { |key, value| !(key =~ /#{sep}([ug]u)?id#{sep}/ && value.is_a?(String)) }
         end
       end
-      request_body_ids = ids_from_body.call(@request_body, request.content_type)
-      data['request']['body_ids'] = request_body_ids if request_body_ids && request_body_ids.any?
-      response_body_string = body.to_enum.to_a.join('')
+      response_body_string = response_body.to_enum.to_a.join('')
       if (400..599).include?(status.to_i)
-        # only log response body if there was an error (either client or server) 
+        # only log bodies if there was an error (either client or server) 
+        data['request']['body'] = request_body
         data['response']['body'] = response_body_string
       else
         # otherwise, log id and uuid fields 
+        request_body_ids = ids_from_body.call(request_body, request.content_type)
+        data['request']['body_ids'] = request_body_ids if request_body_ids && request_body_ids.any?
         response_body_ids = ids_from_body.call(response_body_string, response.content_type)
         data['response']['body_ids'] = response_body_ids if response_body_ids && response_body_ids.any?
       end
@@ -112,8 +130,8 @@ module ApiHammer
         @logger.info json_data
       end
       # do the logging with tags that applied to the request if appropriate 
-      if @logger.respond_to?(:tagged) && @log_tags
-        @logger.tagged(@log_tags, &dolog)
+      if @logger.respond_to?(:tagged) && log_tags
+        @logger.tagged(log_tags, &dolog)
       else
         dolog.call
       end

data/lib/api_hammer/version.rb

@@ -1,3 +1,3 @@
 module ApiHammer
-  VERSION = "0.4.3"
+  VERSION = "0.5.0"
 end

data/test/faraday_request_logger_test.rb

@@ -104,5 +104,25 @@ describe ApiHammer::RequestLogger do
       conn.get '/'
       assert_match('[120,120,195]', logio.string)
     end
+
+    {
+      'application/octet-stream' => false,
+      'image/png' => false,
+      'image/png; charset=what' => false,
+      'text/plain' => true,
+      'text/plain; charset=utf-8' => true,
+    }.each do |content_type, istext|
+      it "does #{istext ? '' : 'not'} log body for #{content_type}" do
+        app = proc do |env|
+          [200, {'Content-Type' => content_type}, ['data go here']]
+        end
+        conn = Faraday.new do |f|
+          f.request :api_hammer_request_logger, logger
+          f.use Faraday::Adapter::Rack, app
+        end
+        conn.get '/'
+        assert(logio.string.include?('data go here') == istext)
+      end
+    end
   end
 end

data/test/request_logger_test.rb

@@ -20,4 +20,25 @@ describe ApiHammer::RequestLogger do
       assert(logio.string.include?(Term::ANSIColor.send(color, status.to_s)))
     end
   end
+
+  it 'logs id and uuid (json)' do
+    body = %q({"uuid": "theuuid", "foo_uuid": "thefoouuid", "id": "theid", "id_for_x": "theidforx", "bar.id": "thebarid", "baz-guid": "bazzz"})
+    app = ApiHammer::RequestLogger.new(proc { |env| [200, {"Content-Type" => 'application/json; charset=UTF8'}, [body]] }, logger)
+    app.call(Rack::MockRequest.env_for('/')).last.close
+    assert_match(%q("body_ids":{"uuid":"theuuid","foo_uuid":"thefoouuid","id":"theid","id_for_x":"theidforx","bar.id":"thebarid","baz-guid":"bazzz"}), logio.string)
+  end
+
+  it 'logs id and uuid (form encoded)' do
+    body = %q(uuid=theuuid&foo_uuid=thefoouuid&id=theid&id_for_x=theidforx&bar.id=thebarid&baz-guid=bazzz)
+    app = ApiHammer::RequestLogger.new(proc { |env| [200, {"Content-Type" => 'application/x-www-form-urlencoded; charset=UTF8'}, [body]] }, logger)
+    app.call(Rack::MockRequest.env_for('/')).last.close
+    assert_match(%q("body_ids":{"uuid":"theuuid","foo_uuid":"thefoouuid","id":"theid","id_for_x":"theidforx","bar.id":"thebarid","baz-guid":"bazzz"}), logio.string)
+  end
+
+  it 'logs request and response body on error' do
+    app = ApiHammer::RequestLogger.new(proc { |env| [400, {}, ['the_response_body']] }, logger)
+    app.call(Rack::MockRequest.env_for('/', :input => 'the_request_body')).last.close
+    assert_match(/"request":\{.*"body":"the_request_body"/, logio.string)
+    assert_match(/"response":\{.*"body":"the_response_body"/, logio.string)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: api_hammer
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.0
 platform: ruby
 authors:
 - Ethan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-23 00:00:00.000000000 Z
+date: 2014-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack