akita-har_logger 0.2.1 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8337c88eacb3b48aa2190bb12b2bc406d1140b6e7d7c9c5ed921b5e92fe557c3
-  data.tar.gz: d6621fcf152e604396634a0614cb9fd66205e95be39be8e46142da90283af8a6
+  metadata.gz: cc8020afb47521c9d7d9ca923dca64e2a8f8364ff7207ab80b68cdc09cd0ca39
+  data.tar.gz: aa865b2768013560e4439d9e500df4680276af28f88294113aa1a594545938eb
 SHA512:
-  metadata.gz: 9efe412f404f5e56abac9af2e79b7d186ab895ceccf40cf369841563f86f5b78eb9ee7fe0cb561b20b7de93a9284343ffa50a355894fe62f6cf3e5d4193ea7d0
-  data.tar.gz: b251114f5be749a02d8d198e2dbce71ff98845085cdb48324c66e1791d3d6acd7252205cce6f5358f4e3a25fbabfecbba392094e21c56a88e8b0d89528beb408
+  metadata.gz: 2f6b7d249bcde1e1d4dea4c332bd1527c1176b7a138ae7dc36edf7844d043e07cea25f845783634fac9385216f746b6209b69e0d4a04eb5b17b2693d5aa9156e
+  data.tar.gz: b58e4e46383ab4caa08b0338cf85b2921c049c7d99df79146756fc2520e267600e1139654059c9ce43faf2807097316a9e874e386fc5c00ee7c2e31e65fb0c0b

data/Gemfile.lock

@@ -1,14 +1,12 @@
 PATH
   remote: .
   specs:
-    akita-har_logger (0.2.1)
-      json (~> 2.3)
+    akita-har_logger (0.2.6)
 
 GEM
   remote: https://rubygems.org/
   specs:
     diff-lcs (1.4.4)
-    json (2.5.1)
     rake (13.0.3)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
@@ -33,4 +31,4 @@ DEPENDENCIES
   rspec (~> 3.10)
 
 BUNDLED WITH
-   2.2.16
+   2.2.23

data/akita-har_logger.gemspec

@@ -31,7 +31,5 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'json', '~> 2.3'
-
   spec.add_development_dependency 'rspec', '~> 3.10'
 end

data/lib/akita/har_logger/har_entry.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'json'
 require_relative 'http_request'
 require_relative 'http_response'
 

data/lib/akita/har_logger/har_utils.rb

@@ -3,17 +3,57 @@
 module Akita
   module HarLogger
     class HarUtils
+      # Rack apparently uses 8-bit ASCII for everything, even when the string
+      # is not 8-bit ASCII. This reinterprets 8-bit ASCII strings as UTF-8.
+      #
+      # If we are unable to do this reinterpretation, return the string
+      # unchanged, but log a warning that points to the caller.
+      def self.fixEncoding(v)
+        if v == nil then
+          return v
+        end
+
+        if !(v.is_a? String) then
+          Rails.logger.warn "[#{caller_locations(1, 1)}] fixEncoding was not given a string. This might cause JSON serialization to fail."
+          return v
+        end
+
+        # Only re-interpret 8-bit ASCII.
+        if v.encoding != Encoding::ASCII_8BIT then
+          return v
+        end
+
+        forced = String.new(v).force_encoding(Encoding::UTF_8)
+        if forced.valid_encoding? then
+          return forced
+        end
+
+        Rails.logger.warn "[#{caller_locations(1, 1)}] Unable to fix encoding: not a valid UTF-8 string. This will likely cause JSON serialization to fail."
+        v
+      end
+
       # Converts a Hash into a list of Hash objects. Each entry in the given
       # Hash will be represented in the output by a Hash object that maps
       # 'name' to the entry's key and 'value' to the entry's value.
       def self.hashToList(hash)
         hash.reduce([]) { |accum, (k, v)|
           accum.append({
-            name: k,
-            value: v,
+            name: fixEncoding(k),
+            value: fixEncoding(v),
           })
         }
       end
+
+      # Determines whether all values in a Hash are strings.
+      def self.allValuesAreStrings(hash)
+        hash.each do |_, value|
+          if !(value.is_a? String) then
+            return false
+          end
+        end
+
+        return true
+      end
     end
   end
 end

data/lib/akita/har_logger/http_request.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'json'
 require_relative 'har_utils'
 
 module Akita
@@ -12,7 +11,7 @@ module Akita
 
         @self = {
           method: getMethod(env),
-          url: req.url,
+          url: HarUtils.fixEncoding(req.url),
           httpVersion: getHttpVersion(env),
           cookies: getCookies(env),
           headers: getHeaders(env),
@@ -34,7 +33,7 @@ module Akita
 
       # Obtains the client's request method from an HTTP environment.
       def getMethod(env)
-        (Rack::Request.new env).request_method
+        HarUtils.fixEncoding (Rack::Request.new env).request_method
       end
 
       # Obtains the client-requested HTTP version from an HTTP environment.
@@ -42,7 +41,9 @@ module Akita
         # The environment doesn't have HTTP_VERSION when running with `rspec`;
         # assume HTTP/1.1 when this happens. We don't return nil, so we can
         # calculate the size of the headers.
-        env.key?('HTTP_VERSION') ? env['HTTP_VERSION'] : 'HTTP/1.1'
+        env.key?('HTTP_VERSION') ?
+          HarUtils.fixEncoding(env['HTTP_VERSION']) :
+          'HTTP/1.1'
       end
 
       # Builds a list of cookie objects from an HTTP environment.
@@ -72,23 +73,71 @@ module Akita
         HarUtils.hashToList paramMap
       end
 
+      # Obtains the character set of the posted data from an HTTP environment.
+      def getPostDataCharSet(env)
+        req = Rack::Request.new env
+        if req.content_charset != nil then
+          return req.content_charset
+        end
+
+        # RFC 2616 says that "text/*" defaults to ISO-8859-1.
+        if env['CONTENT_TYPE'].start_with?('text/') then
+          return Encoding::ISO_8859_1
+        end
+
+        Encoding::UTF_8
+      end
+
       # Obtains the posted data from an HTTP environment.
       def getPostData(env)
         if env.key?('CONTENT_TYPE') && env['CONTENT_TYPE'] then
           result = { mimeType: env['CONTENT_TYPE'] }
 
           # Populate 'params' if we have URL-encoded parameters. Otherwise,
-          # populate 'text.
+          # populate 'text'.
           req = Rack::Request.new env
           if env['CONTENT_TYPE'] == 'application/x-www-form-urlencoded' then
-            # Decoded parameters can be found as a map in req.params. Convert
-            # this map into an array.
+            # Decoded parameters can be found as a map in req.params.
             #
-            # XXX Spec has space for files, but are file uploads ever
-            # URL-encoded?
-            result[:params] = HarUtils.hashToList req.params
+            # Requests originating from specs can be malformed: the values in
+            # req.params are not necessarily strings. Encode all of req.params
+            # in JSON and pretend the content type was "application/json".
+            if HarUtils.allValuesAreStrings req.params then
+              # Convert req.params into an array.
+              #
+              # XXX Spec has space for files, but are file uploads ever
+              # URL-encoded?
+              result[:params] = HarUtils.hashToList req.params
+            else
+              result[:mimeType] = 'application/json'
+              result[:text] = req.params.to_json
+            end
           else
-            result[:text] = req.body.string
+            # Rack has been observed to use ASCII-8BIT encoding for the request
+            # body when the request specifies UTF-8. Reinterpret the content
+            # body according to what the request says it is, and re-encode into
+            # UTF-8.
+            #
+            # Gracefully handle any characters that are invalid in the source
+            # encoding and characters that have no UTF-8 representation by
+            # replacing with '?'. Log a warning when this happens.
+            source = req.body.string.force_encoding(getPostDataCharSet(env))
+            utf8EncodingSuccessful = false
+            if source.valid_encoding? then
+              begin
+                result[:text] = source.encode(Encoding::UTF_8)
+                utf8EncodingSuccessful = true
+              rescue Encoding::UndefinedConversionError
+                Rails.logger.warn "[#{caller_locations(0, 1)}] Unable to losslessly convert request body from #{source.encoding} to UTF-8. Characters undefined in UTF-8 will be replaced with '?'."
+              end
+            else
+              Rails.logger.warn "[#{caller_locations(0, 1)}] Request body is not valid #{source.encoding}. Invalid characters and characters undefined in UTF-8 will be replaced with '?'."
+            end
+
+            if !utf8EncodingSuccessful then
+              result[:text] = source.encode(Encoding::UTF_8,
+                  invalid: :replace, undef: :replace, replace: '?')
+            end
           end
 
           result

data/lib/akita/har_logger/http_response.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'json'
 require_relative 'har_utils'
 
 module Akita
@@ -26,7 +25,7 @@ module Akita
 
       # Obtains the status text corresponding to a status code.
       def getStatusText(status)
-        Rack::Utils::HTTP_STATUS_CODES[status]
+        HarUtils.fixEncoding(Rack::Utils::HTTP_STATUS_CODES[status])
       end
 
       # Obtains the HTTP version in the response.
@@ -37,7 +36,9 @@ module Akita
         # The environment doesn't have HTTP_VERSION when running with `rspec`;
         # assume HTTP/1.1 when this happens. We don't return nil, so we can
         # calculate the size of the headers.
-        env.key?('HTTP_VERSION') ? env['HTTP_VERSION'] : 'HTTP/1.1'
+        env.key?('HTTP_VERSION') ?
+          HarUtils.fixEncoding(env['HTTP_VERSION']) :
+          'HTTP/1.1'
       end
 
       def getCookies(headers)
@@ -64,8 +65,8 @@ module Akita
           if match then cookie_value = match[1] end
 
           result << {
-            name: cookie_name,
-            value: cookie_value,
+            name: HarUtils.fixEncoding(cookie_name),
+            value: HarUtils.fixEncoding(cookie_value),
           }
         }
 
@@ -73,19 +74,56 @@ module Akita
       end
 
       def getContent(headers, body)
-        # XXX Handle compression & encoding.
+        # XXX Handle compression
+        # XXX Figure out how to properly join together multi-part bodies.
 
+        # Try to convert the body into UTF-8. If this fails, assume the body is
+        # binary data.
+        # XXX TODO Take charset part of Content-Type header into account.
         text = +""
+        haveBinaryData = false
         body.each { |part|
-          # XXX Figure out how to join together multi-part bodies.
-          text << part;
+          partStr = part.to_s
+
+          if partStr.encoding == Encoding::ASCII_8BIT then
+            # Have 8-bit ASCII data. Try to interpret as UTF-8. If this fails,
+            # treat as binary data.
+            forced = String.new(partStr).force_encoding(Encoding::UTF_8)
+            if forced.valid_encoding? then
+              text << forced
+              next
+            end
+
+            haveBinaryData = true
+            break
+          end
+
+          if !partStr.valid_encoding? then
+            # Source encoding is not valid. Treat as binary data.
+            haveBinaryData = true
+            break
+          end
+
+          # Try to re-encode as UTF-8. If this fails, treat as binary data.
+          begin
+            text << partStr.encode(Encoding::UTF_8)
+          rescue Encoding::UndefinedConversionError
+            haveBinaryData = true
+            break
+          end
         }
 
+        if haveBinaryData then
+          # TODO Encode binary body data with base64.
+          # XXX Omit for now.
+          text = ""
+        end
+
         {
           size: getBodySize(body),
 
           # XXX What to use when no Content-Type is given?
-          mimeType: headers['Content-Type'],
+          mimeType: HarUtils.fixEncoding(headers['Content-Type']),
 
           text: text,
         }
@@ -94,7 +132,9 @@ module Akita
       def getRedirectUrl(headers)
         # Use the "Location" header if it exists. Otherwise, based on some HAR
         # examples found online, it looks like an empty string is used.
-        headers.key?('Location') ? headers['Location'] : ''
+        headers.key?('Location') ?
+          HarUtils.fixEncoding(headers['Location']) :
+          ''
       end
 
       def getHeadersSize(env, status, headers)

data/lib/akita/har_logger/version.rb

@@ -2,6 +2,6 @@
 
 module Akita
   module HarLogger
-    VERSION = "0.2.1"
+    VERSION = "0.2.6"
   end
 end

data/lib/akita/har_logger/writer_thread.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'json'
-
 module Akita
   module HarLogger
     # A thread that consumes HarEntry objects from a queue and writes them to a

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: akita-har_logger
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.6
 platform: ruby
 authors:
 - Jed Liu
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-04-21 00:00:00.000000000 Z
+date: 2021-07-13 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: json
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -83,7 +69,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.2.21
 signing_key:
 specification_version: 4
 summary: Rails middleware for HAR logging