api_hammer 0.10.2 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e22063457c4fd23dcfacf1dd263bb86bed720da4
-  data.tar.gz: 59d655dfeb7c3521bda18fa73cc58da1a0cee266
+  metadata.gz: 6d475db83c102fc0746ea2da97c992a9ab1e6a1f
+  data.tar.gz: f82cbddaf738a334f7779bed1ce64005c6e46f93
 SHA512:
-  metadata.gz: e26ef7aec22dc8114044bb52947d3d1a49e00395ae45c80feb6903487385f3db6bf4f0871c32558e943cc7a77d85358d9d705695c9ae2a5acec9d25ad41eb76e
-  data.tar.gz: 7b0d75819bfd4c637d24a0ee74cea82e589b3ac3d5f767129a7ac43b4370cb44fb0d4df01d5398f7324b5f4a1805a4cbc6727a19456f1c5cc2bb34373b1bec09
+  metadata.gz: 42161c34bc9260717cba35c2ce308636490e332722e43d1435fa2300b404f97156701d9bed860a727ed4dbf2fcab010968f6cb360d33dd10b63dfaf4f73b7d75
+  data.tar.gz: 59f6ccc971e7d4c925a7141c7fa91670bf36e83972598c0583015c0df5c48759195bf34fb6499887b3703fce63f9610a700f9d91b334720e9627bf93792ac0cf

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# v0.11.0
+- improved handling of text and binary bodies in logging middleware and hc
+
 # v0.10.2
 - rails request logging logs exception backtrace
 

data/bin/hc

@@ -44,6 +44,9 @@ opt_parser = OptionParser.new do |opts|
   opts.on("-t", "--content-type CONTENT-TYPE", "Sets the Content-Type header of the request. This defaults to application/json if a body is included.") do |v|
     $options[:headers]['Content-Type'.downcase] = v
   end
+  opts.on("-o", "--output OUTPUT", "write response to file") do |v|
+    $options[:output] = v
+  end
   opts.on("--oauth-token TOKEN", "OAuth 1.0 token") do |token|
     $oauth[:token] = token
   end
@@ -116,3 +119,9 @@ end
 # OH LOOK IT'S FINALLY ACTUALLY CONNECTING TO SOMETHING
 
 response = connection.run_request(httpmethod.downcase.to_sym, url, body, headers)
+
+if $options[:output]
+  File.open($options[:output], 'wb') do |f|
+    f.write(response.body)
+  end
+end

data/lib/api_hammer/body.rb

@@ -0,0 +1,103 @@
+require 'rack'
+
+module ApiHammer
+  class Body
+    attr_reader :body, :content_type
+
+    def initialize(body, content_type)
+      @body = body
+      @content_type = content_type
+    end
+
+    # parses the body to an object
+    def object
+      instance_variable_defined?(:@object) ? @object : @object = begin
+        if media_type == 'application/json'
+          JSON.parse(body) rescue nil
+        elsif media_type == 'application/x-www-form-urlencoded'
+          CGI.parse(body).map { |k, vs| {k => vs.last} }.inject({}, &:update)
+        end
+      end
+    end
+
+    def filtered(options)
+      @filtered ||= Body.new(begin
+        if media_type == 'application/json'
+          begin
+            ApiHammer::Filtration::Json.new(body, options).filter
+          rescue JSON::ParserError
+            body
+          end
+        elsif media_type == 'application/x-www-form-urlencoded'
+          ApiHammer::Filtration::FormEncoded.new(body, options).filter
+        else
+          body
+        end
+      end, content_type)
+    end
+
+    def content_type_attrs
+      @content_type_attrs ||= ContentTypeAttrs.new(content_type)
+    end
+
+    def media_type
+      content_type_attrs.media_type
+    end
+
+    # deal with the vagaries of getting the response body in a form which JSON 
+    # gem will not cry about generating 
+    def jsonifiable
+      @jsonifiable ||= Body.new(catch(:jsonifiable) do
+        original_body = self.body
+        unless original_body.is_a?(String)
+          begin
+            # if the response body is not a string, but JSON doesn't complain 
+            # about dumping whatever it is, go ahead and use it
+            JSON.generate([original_body])
+            throw :jsonifiable, original_body
+          rescue
+            # otherwise return nil - don't know what to do with whatever this object is 
+            throw :jsonifiable, nil
+          end
+        end
+
+        # first try to change the string's encoding per the Content-Type header 
+        body = original_body.dup
+        unless body.valid_encoding?
+          # I think this always comes in as ASCII-8BIT anyway so may never get here. hopefully.
+          body.force_encoding('ASCII-8BIT')
+        end
+
+        content_type_attrs = ContentTypeAttrs.new(content_type)
+        if content_type_attrs.parsed?
+          charset = content_type_attrs['charset'].first
+          if charset && Encoding.list.any? { |enc| enc.to_s.downcase == charset.downcase }
+            if body.dup.force_encoding(charset).valid_encoding?
+              body.force_encoding(charset)
+            else
+              # I guess just ignore the specified encoding if the result is not valid. fall back to 
+              # something else below.
+            end
+          end
+        end
+        begin
+          JSON.generate([body])
+        rescue Encoding::UndefinedConversionError
+          # if updating by content-type didn't do it, try UTF8 since JSON wants that - but only 
+          # if it seems to be valid utf8. 
+          # don't try utf8 if the response content-type indicated something else. 
+          try_utf8 = !(content_type_attrs && content_type_attrs.parsed? && content_type_attrs['charset'].any?)
+          if try_utf8 && body.dup.force_encoding('UTF-8').valid_encoding?
+            body.force_encoding('UTF-8')
+          else
+            # I'm not sure if there is a way in this situation to get JSON gem to generate the 
+            # string correctly. fall back to an array of codepoints I guess? this is a weird 
+            # solution but the best I've got for now. 
+            body = body.codepoints.to_a
+          end
+        end
+        body
+      end, content_type)
+    end
+  end
+end

data/lib/api_hammer/content_type_attrs.rb

@@ -27,7 +27,7 @@ module ApiHammer
       end
     end
 
-    attr_reader :media_type
+    attr_reader :content_type, :media_type
 
     def parsed?
       @parsed
@@ -47,12 +47,19 @@ module ApiHammer
         %r(model/.*) => :binary,
         %r(text/.*) => :text,
         %r(message/.*) => :text,
+        %r(application/(.+\+)?json) => :text,
+        %r(application/(.+\+)?xml) => :text,
+        %r(model/(.+\+)?xml) => :text,
+        'application/x-www-form-urlencoded' => :text,
+        'application/javascript' => :text,
+        'application/ecmascript' => :text,
         'application/octet-stream' => :binary,
         'application/ogg' => :binary,
         'application/pdf' => :binary,
         'application/postscript' => :binary,
         'application/zip' => :binary,
         'application/gzip' => :binary,
+        'application/vnd.apple.pkpass' => :binary,
       }
       types.each do |match, type|
         matched = match.is_a?(Regexp) ? media_type =~ %r(\A#{match.source}\z) : media_type == match
@@ -60,8 +67,13 @@ module ApiHammer
           return type == :text
         end
       end
-      # fallback (unknown or not given) assume text
-      return true
+      # fallback (unknown or not given) assume that unknown content types are binary but omitted 
+      # content-type means text
+      if content_type && content_type =~ /\S/
+        return false
+      else
+        return true
+      end
     end
   end
 end

data/lib/api_hammer/faraday/outputter.rb

@@ -1,5 +1,6 @@
 require 'faraday'
 require 'rack'
+require 'api_hammer'
 
 module ApiHammer
   # outputs the response body to the given logger or output device (defaulting to STDOUT) 
@@ -55,6 +56,8 @@ module ApiHammer
     color :response_header
     color :response_blankline, :intense_green, :bold
 
+    color :omitted_body, :intense_yellow
+
     def call(request_env)
       puts "#{info('*')} #{info_body("connect to #{request_env[:url].host} on port #{request_env[:url].port}")}"
       puts "#{info('*')} #{info_body("getting our SSL on")}" if request_env[:url].scheme=='https'
@@ -125,23 +128,27 @@ module ApiHammer
     # - formatted prettily if #pretty? is true
     def alter_body_by_content_type(body, content_type)
       return body unless body.is_a?(String)
-      media_type = ::Rack::Request.new({'CONTENT_TYPE' => content_type}).media_type
-      if pretty?
-        case media_type
-        when 'application/json'
-          require 'json'
-          begin
-            body = JSON.pretty_generate(JSON.parse(body))
-          rescue JSON::ParserError
+      content_type_attrs = ApiHammer::ContentTypeAttrs.new(content_type)
+      if content_type_attrs.text?
+        if pretty?
+          case content_type_attrs.media_type
+          when 'application/json'
+            require 'json'
+            begin
+              body = JSON.pretty_generate(JSON.parse(body))
+            rescue JSON::ParserError
+            end
           end
         end
-      end
-      if color?
-        coderay_scanner = CodeRayForMediaTypes.reject{|k,v| !v.any?{|type| type === media_type} }.keys.first
-        if coderay_scanner
-          require 'coderay'
-          body = CodeRay.scan(body, coderay_scanner).encode(:terminal)
+        if color?
+          coderay_scanner = CodeRayForMediaTypes.reject{|k,v| !v.any?{|type| type === content_type_attrs.media_type} }.keys.first
+          if coderay_scanner
+            require 'coderay'
+            body = CodeRay.scan(body, coderay_scanner).encode(:terminal)
+          end
         end
+      else
+        body = omitted_body("[[omitted binary body (size = #{body.size})]]")
       end
       body
     end

data/lib/api_hammer/faraday/request_logger.rb

@@ -5,72 +5,6 @@ require 'strscan'
 
 module ApiHammer
   module Faraday
-    class Request
-      def initialize(request_env, response_env)
-        @request_env = request_env
-        @response_env = response_env
-      end
-
-      attr_reader :request_env
-      attr_reader :response_env
-
-      # deal with the vagaries of getting the response body in a form which JSON 
-      # gem will not cry about generating 
-      def response_body
-        instance_variable_defined?(:@response_body) ? @response_body : @response_body = catch(:response_body) do
-          unless response_env.body.is_a?(String)
-            begin
-              # if the response body is not a string, but JSON doesn't complain 
-              # about dumping whatever it is, go ahead and use it
-              JSON.generate([response_env.body])
-              throw :response_body, response_env.body
-            rescue
-              # otherwise return nil - don't know what to do with whatever this object is 
-              throw :response_body, nil
-            end
-          end
-
-          # first try to change the string's encoding per the Content-Type header 
-          content_type = response_env.response_headers['Content-Type']
-          response_body = response_env.body.dup
-          unless response_body.valid_encoding?
-            # I think this always comes in as ASCII-8BIT anyway so may never get here. hopefully.
-            response_body.force_encoding('ASCII-8BIT')
-          end
-
-          content_type_attrs = ContentTypeAttrs.new(content_type)
-          if content_type_attrs.parsed?
-            charset = content_type_attrs['charset'].first
-            if charset && Encoding.list.any? { |enc| enc.to_s.downcase == charset.downcase }
-              if response_body.dup.force_encoding(charset).valid_encoding?
-                response_body.force_encoding(charset)
-              else
-                # I guess just ignore the specified encoding if the result is not valid. fall back to 
-                # something else below.
-              end
-            end
-          end
-          begin
-            JSON.generate([response_body])
-          rescue Encoding::UndefinedConversionError
-            # if updating by content-type didn't do it, try UTF8 since JSON wants that - but only 
-            # if it seems to be valid utf8. 
-            # don't try utf8 if the response content-type indicated something else. 
-            try_utf8 = !(content_type_attrs && content_type_attrs.parsed? && content_type_attrs['charset'].any?)
-            if try_utf8 && response_body.dup.force_encoding('UTF-8').valid_encoding?
-              response_body.force_encoding('UTF-8')
-            else
-              # I'm not sure if there is a way in this situation to get JSON gem to generate the 
-              # string correctly. fall back to an array of codepoints I guess? this is a weird 
-              # solution but the best I've got for now. 
-              response_body = response_body.codepoints.to_a
-            end
-          end
-          response_body
-        end
-      end
-    end
-
     # Faraday middleware for logging.
     #
     # logs two lines:
@@ -101,8 +35,6 @@ module ApiHammer
         @app.call(request_env).on_complete do |response_env|
           now = Time.now
 
-          request = ApiHammer::Faraday::Request.new(request_env, response_env)
-
           status_color = case response_env.status.to_i
           when 200..299
             :intense_green
@@ -115,17 +47,17 @@ module ApiHammer
           end
           status_s = bold(send(status_color, response_env.status.to_s))
 
-          filtered_request_body = request_body.dup if request_body
-          filtered_response_body = request.response_body.dup if request.response_body
+          bodies = [
+            ['request', request_body, request_env.request_headers],
+            ['response', response_env.body, response_env.response_headers]
+          ].map do |(role, body, headers)|
+            {role => Body.new(body, headers['Content-Type']).jsonifiable}
+          end.inject({}, &:update)
 
           if @options[:filter_keys]
-            body_info = [['request', filtered_request_body, request_env.request_headers], ['response', filtered_response_body, response_env.response_headers]]
-            body_info.map do |(role, body, headers)|
-              if body
-                parsed_body = ApiHammer::ParsedBody.new(body, headers['Content-Type'])
-                body.replace(parsed_body.filtered_body(@options.slice(:filter_keys)))
-              end
-            end
+            bodies = bodies.map do |(role, body)|
+              {role => body.filtered(@options.slice(:filter_keys))}
+            end.inject({}, &:update)
           end
 
           data = {
@@ -134,12 +66,12 @@ module ApiHammer
               'method' => request_env[:method],
               'uri' => request_env[:url].normalize.to_s,
               'headers' => request_env.request_headers,
-              'body' => (filtered_request_body if ContentTypeAttrs.new(request_env.request_headers['Content-Type']).text?),
+              'body' => (bodies['request'].body if bodies['request'].content_type_attrs.text?),
             }.reject{|k,v| v.nil? },
             'response' => {
               'status' => response_env.status.to_s,
               'headers' => response_env.response_headers,
-              'body' => (filtered_response_body if ContentTypeAttrs.new(response_env.response_headers['Content-Type']).text?),
+              'body' => (bodies['response'].body if bodies['response'].content_type_attrs.text?),
             }.reject{|k,v| v.nil? },
             'processing' => {
               'began_at' => began_at.utc.to_f,

data/lib/api_hammer/request_logger.rb

@@ -112,24 +112,27 @@ module ApiHammer
       response_body_string = response_body.to_enum.to_a.join('')
       body_info = [['request', request_body, request.content_type], ['response', response_body_string, response.content_type]]
       body_info.map do |(role, body, content_type)|
-        parsed_body = ApiHammer::ParsedBody.new(body, content_type)
-        if (400..599).include?(status.to_i) || body.size < LARGE_BODY_SIZE
-          # log bodies if they are not large, or if there was an error (either client or server) 
-          data[role]['body'] = parsed_body.filtered_body(@options.reject { |k,v| ![:filter_keys].include?(k) })
-        else
-          # otherwise, log id and uuid fields 
-          body_object = parsed_body.object
-          sep = /(?:\b|\W|_)/
-          hash_ids = proc do |hash|
-            hash.reject { |key, value| !(key =~ /#{sep}([ug]u)?id#{sep}/ && value.is_a?(String)) }
-          end
-          if body_object.is_a?(Hash)
-            body_ids = hash_ids.call(body_object)
-          elsif body_object.is_a?(Array) && body_object.all? { |e| e.is_a?(Hash) }
-            body_ids = body_object.map(&hash_ids)
-          end
+        parsed_body = ApiHammer::Body.new(body, content_type)
+        content_type_attrs = ApiHammer::ContentTypeAttrs.new(content_type)
+        if content_type_attrs.text?
+          if (400..599).include?(status.to_i) || body.size < LARGE_BODY_SIZE
+            # log bodies if they are not large, or if there was an error (either client or server) 
+            data[role]['body'] = parsed_body.jsonifiable.filtered(@options.reject { |k,v| ![:filter_keys].include?(k) }).body
+          else
+            # otherwise, log id and uuid fields 
+            body_object = parsed_body.object
+            sep = /(?:\b|\W|_)/
+            hash_ids = proc do |hash|
+              hash.reject { |key, value| !(key =~ /#{sep}([ug]u)?id#{sep}/ && value.is_a?(String)) }
+            end
+            if body_object.is_a?(Hash)
+              body_ids = hash_ids.call(body_object)
+            elsif body_object.is_a?(Array) && body_object.all? { |e| e.is_a?(Hash) }
+              body_ids = body_object.map(&hash_ids)
+            end
 
-          data[role]['body_ids'] = body_ids if body_ids && body_ids.any?
+            data[role]['body_ids'] = body_ids if body_ids && body_ids.any?
+          end
         end
       end
       Thread.current['request_logger.info'] = nil

data/lib/api_hammer/trailing_newline.rb

@@ -26,7 +26,7 @@ module ApiHammer
     end
     def call(env)
       status, headers, body = *@app.call(env)
-      if env['REQUEST_METHOD'].downcase != 'head'
+      if env['REQUEST_METHOD'].downcase != 'head' && ApiHammer::ContentTypeAttrs.new(env['CONTENT_TYPE']).text?
         body = TNLBodyProxy.new(body){}
         if headers["Content-Length"]
           headers["Content-Length"] = body.map(&Rack::Utils.method(:bytesize)).inject(0, &:+).to_s

data/lib/api_hammer/version.rb

@@ -1,3 +1,3 @@
 module ApiHammer
-  VERSION = "0.10.2"
+  VERSION = "0.11.0"
 end

data/lib/api_hammer.rb

@@ -12,7 +12,7 @@ module ApiHammer
   autoload :RailsOrSidekiqLogger, 'api_hammer/rails_or_sidekiq_logger'
   autoload :FaradayOutputter, 'api_hammer/faraday/outputter'
   autoload :FaradayCurlVOutputter, 'api_hammer/faraday/outputter'
-  autoload :ParsedBody, 'api_hammer/parsed_body'
+  autoload :Body, 'api_hammer/body'
   autoload :ContentTypeAttrs, 'api_hammer/content_type_attrs'
   autoload :JsonScriptEscapeHelper, 'api_hammer/json_script_escape_helper'
   module Faraday

data/lib/logstash/filters/request_bodies_parsed.rb

@@ -21,7 +21,7 @@ class LogStash::Filters::RequestBodiesParsed < LogStash::Filters::Base
         if event[re]['headers'].is_a?(Hash) && !content_type
           _, content_type = event[re]['headers'].detect { |(k,_)| k =~ /\Acontent.type\z/i }
         end
-        parsed_body = ApiHammer::ParsedBody.new(event[re]['body'], content_type)
+        parsed_body = ApiHammer::Body.new(event[re]['body'], content_type)
         event[re]['body_parsed'] = parsed_body.object if parsed_body.object
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: api_hammer
 version: !ruby/object:Gem::Version
-  version: 0.10.2
+  version: 0.11.0
 platform: ruby
 authors:
 - Ethan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-19 00:00:00.000000000 Z
+date: 2015-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -244,6 +244,7 @@ files:
 - bin/hc
 - lib/api_hammer.rb
 - lib/api_hammer/active_record_cache_find_by.rb
+- lib/api_hammer/body.rb
 - lib/api_hammer/check_required_params.rb
 - lib/api_hammer/content_type_attrs.rb
 - lib/api_hammer/faraday/outputter.rb
@@ -252,7 +253,6 @@ files:
 - lib/api_hammer/filtration/json.rb
 - lib/api_hammer/halt.rb
 - lib/api_hammer/json_script_escape_helper.rb
-- lib/api_hammer/parsed_body.rb
 - lib/api_hammer/public_instance_exec.rb
 - lib/api_hammer/rails.rb
 - lib/api_hammer/rails_or_sidekiq_logger.rb

data/lib/api_hammer/parsed_body.rb

@@ -1,39 +0,0 @@
-require 'rack'
-
-module ApiHammer
-  class ParsedBody
-    attr_reader :body, :content_type, :media_type
-
-    def initialize(body, content_type)
-      @body = body
-      @content_type = content_type
-      @media_type = ::Rack::Request.new({'CONTENT_TYPE' => content_type}).media_type
-    end
-
-    def object
-      instance_variable_defined?(:@object) ? @object : @object = begin
-        if media_type == 'application/json'
-          JSON.parse(body) rescue nil
-        elsif media_type == 'application/x-www-form-urlencoded'
-          CGI.parse(body).map { |k, vs| {k => vs.last} }.inject({}, &:update)
-        end
-      end
-    end
-
-    def filtered_body(options)
-      @filtered_body ||= begin
-        if media_type == 'application/json'
-          begin
-            ApiHammer::Filtration::Json.new(body, options).filter
-          rescue JSON::ParserError
-            body
-          end
-        elsif media_type == 'application/x-www-form-urlencoded'
-          ApiHammer::Filtration::FormEncoded.new(body, options).filter
-        else
-          body
-        end
-      end
-    end
-  end
-end