timber 2.5.1 → 2.6.0.pre.beta1

data/lib/timber/util/hash.rb

@@ -2,6 +2,7 @@ module Timber
   module Util
     # @private
     module Hash
+      BINARY_LIMIT_THRESHOLD = 1_000.freeze
       SANITIZED_VALUE = '[sanitized]'.freeze
 
       extend self
@@ -25,7 +26,37 @@ module Timber
         new_hash
       end
 
-      def sanitize(hash, keys_to_sanitize)
+      # Recursively traverses a hash, dropping non-JSON compatible types.
+      # If the string is a binary, and it is > 1000 characters, it is dropped.
+      # We are assuming it represents file contents that should not be included
+      # in the logs.
+      def jsonify(hash)
+        deep_reduce(hash) do |k, v, h|
+          if v.is_a?(String)
+            if v.encoding == ::Encoding::ASCII_8BIT
+              # Only keep binary values less than a certain size. Sizes larger than this
+              # are almost always file uploads and data we do not want to log.
+              if v.length < BINARY_LIMIT_THRESHOLD
+                # Attempt to safely encode the data to UTF-8
+                encoded_value = encode_string(v)
+                if !encoded_value.nil?
+                  h[k] = encoded_value
+                end
+              end
+            elsif v.encoding != ::Encoding::UTF_8
+              h[k] = encode_string(v)
+            else
+              h[k] = v
+            end
+          elsif is_a_primitive_type?(v)
+            # Keep all other primitive types
+            h[k] = v
+          end
+        end
+      end
+
+      # Replaces matching keys with a `[Sanitized]` value.
+      def sanitize_keys(hash, keys_to_sanitize)
         hash.each_with_object({}) do |(k, v), h|
           k = k.to_s.downcase
           if keys_to_sanitize.include?(k)
@@ -35,6 +66,25 @@ module Timber
           end
         end
       end
+
+      private
+        # Attempts to encode a non UTF-8 string into UTF-8, discarding invalid characters.
+        # If it fails, a nil is returned.
+        def encode_string(string)
+          string.encode('UTF-8', {
+            :invalid => :replace,
+            :undef   => :replace,
+            :replace => '?'
+          })
+        rescue Exception
+          nil
+        end
+
+        # We use is_a? because it accounts for inheritance.
+        def is_a_primitive_type?(v)
+          v.is_a?(Array) || v.is_a?(Integer) || v.is_a?(Float) || v.is_a?(TrueClass) ||
+            v.is_a?(FalseClass) || v.is_a?(String) || v.is_a?(Time)
+        end
     end
   end
 end

data/lib/timber/util/non_nil_hash_builder.rb

@@ -0,0 +1,38 @@
+module Timber
+  module Util
+    # @private
+    #
+    # The purpose of this class is to efficiently build a hash that does not
+    # include nil values. It's proactive instead of reactive, avoiding the
+    # need to traverse and reduce a new hash dropping blanks.
+    class NonNilHashBuilder
+      class << self
+        def build(&block)
+          builder = new
+          yield builder
+          builder.target
+        end
+      end
+
+      attr_reader :target
+
+      def initialize
+        @target = {}
+      end
+
+      def add(k, v, options = {})
+        if !v.nil?
+          if options[:json_encode]
+            v = v.to_json
+          end
+
+          if options[:limit]
+            v = v.byteslice(0, options[:limit])
+          end
+
+          @target[k] = v
+        end
+      end
+    end
+  end
+end

data/lib/timber/version.rb

@@ -1,3 +1,3 @@
 module Timber
-  VERSION = "2.5.1"
+  VERSION = "2.6.0-beta1"
 end

data/spec/timber/events/error_spec.rb

@@ -1,34 +1,20 @@
 require "spec_helper"
 
 describe Timber::Events::Error, :rails_23 => true do
-  describe ".initialize" do
-    it "should clean the backtrace" do
+  describe "#to_hash" do
+    it "should jsonify the stacktrace" do
       backtrace = [
         "/path/to/file1.rb:26:in `function1'",
         "path/to/file2.rb:86:in `function2'"
       ]
-
       exception_event = described_class.new(name: "RuntimeError", error_message: "Boom", backtrace: backtrace)
-      expect(exception_event.backtrace).to eq([{:file=>"/path/to/file1.rb", :line=>26, :function=>"function1"}, {:file=>"path/to/file2.rb", :line=>86, :function=>"function2"}])
-    end
-
-    it "parses valid lines" do
-      backtrace = [
-        "/path/to/file1.rb:26:in `function1'",
-        "path/to/file2.rb:86" # function names are optional
-      ]
 
-      exception_event = described_class.new(name: "RuntimeError", error_message: "Boom", backtrace: backtrace)
-      expect(exception_event.backtrace).to eq([{:file=>"/path/to/file1.rb", :line=>26, :function=>"function1"}, {:file=>"path/to/file2.rb", :line=>86}])
+      expected_hash = {
+        :name => "RuntimeError",
+        :message => "Boom",
+        :backtrace_json => "[\"/path/to/file1.rb:26:in `function1'\",\"path/to/file2.rb:86:in `function2'\"]"
+      }
+      expect(exception_event.to_hash).to eq(expected_hash)
     end
-
-    it "handles malformed lines" do
-      backtrace = [
-        "malformed"
-      ]
-
-      exception_event = described_class.new(name: "RuntimeError", error_message: "Boom", backtrace: backtrace)
-      expect(exception_event.backtrace).to eq([{:file=>"malformed"}])
-   end
   end
 end

data/spec/timber/events/http_request_spec.rb

@@ -26,7 +26,7 @@ describe Timber::Events::HTTPRequest, :rails_23 => true do
     it "should handle header encoding" do
       referer = 'http://www.metrojobb.se/jobb/1013893-skadeadministratör'.force_encoding('ASCII-8BIT')
       event = described_class.new(:headers => {'Referer' => referer}, :host => 'my.host.com', :method => 'GET', :path => '/path', :scheme => 'https')
-      expect(event.headers.key?("referer")).to eq(false)
+      expect(event.headers["referer"].encoding).to eq(::Encoding::UTF_8)
     end
   end
 end

data/spec/timber/events_spec.rb

@@ -18,7 +18,7 @@ describe Timber::Events, :rails_23 => true do
           Timber::Events::Custom.new(
             type: :payment_rejected,
             message: "Payment rejected for #{customer_id}",
-            data: respond_to?(:hash) ? hash : to_hash
+            data: respond_to?(:to_h) ? to_h : hash
           )
         end
       end

data/spec/timber/integrations/action_dispatch/debug_exceptions_spec.rb

@@ -41,7 +41,7 @@ if defined?(::ActionDispatch)
         lines = clean_lines(io.string.split("\n"))
         expect(lines.length).to eq(3)
         expect(lines[2]).to start_with('RuntimeError (boom) @metadata {"level":"fatal",')
-        expect(lines[2]).to include("\"event\":{\"error\":{\"name\":\"RuntimeError\",\"message\":\"boom\",\"backtrace\":[")
+        expect(lines[2]).to include("\"event\":{\"error\":{\"name\":\"RuntimeError\",\"message\":\"boom\",\"backtrace_json\":\"[")
       end
 
       # Remove blank lines since Rails does this to space out requests in the logs

data/spec/timber/log_entry_spec.rb

@@ -3,45 +3,6 @@ require "spec_helper"
 describe Timber::LogEntry, :rails_23 => true do
   let(:time) { Time.utc(2016, 9, 1, 12, 0, 0) }
 
-  describe "#as_json" do
-    it "should drop nil value keys" do
-      event = Timber::Events::Custom.new(type: :event_type, message: "event_message", data: {a: nil})
-      log_entry = described_class.new("INFO", time, nil, "log message", {}, event)
-      hash = log_entry.as_json
-      expect(hash.key?(:event)).to be false
-    end
-
-    it "should drop blank string value keys" do
-      event = Timber::Events::Custom.new(type: :event_type, message: "event_message", data: {a: ""})
-      log_entry = described_class.new("INFO", time, nil, "log message", {}, event)
-      hash = log_entry.as_json
-      expect(hash.key?(:event)).to be false
-    end
-
-    it "should drop empty array value keys" do
-      event = Timber::Events::Custom.new(type: :event_type, message: "event_message", data: {a: []})
-      log_entry = described_class.new("INFO", time, nil, "log message", {}, event)
-      hash = log_entry.as_json
-      expect(hash.key?(:event)).to be false
-    end
-
-    it "should drop ascii-8bit (binary) value keys" do
-      binary = ("a" * 1001).force_encoding("ASCII-8BIT")
-      event = Timber::Events::Custom.new(type: :event_type, message: "event_message", data: {a: binary})
-      log_entry = described_class.new("INFO", time, nil, "log message", {}, event)
-      hash = log_entry.as_json
-      expect(hash.key?(:event)).to be false
-    end
-
-    it "should keep ascii-8bit (binary) values below the threshold" do
-      binary = "test".force_encoding("ASCII-8BIT")
-      event = Timber::Events::Custom.new(type: :event_type, message: "event_message", data: {a: binary})
-      log_entry = described_class.new("INFO", time, nil, "log message", {}, event)
-      hash = log_entry.as_json
-      expect(hash[:event][:custom][:event_type][:a].encoding).to eq(::Encoding::UTF_8)
-    end
-  end
-
   describe "#to_msgpack" do
     it "should encode properly with an event and context" do
       event = Timber::Events::Custom.new(type: :event_type, message: "event_message", data: {a: 1})

data/spec/timber/logger_spec.rb

@@ -130,11 +130,6 @@ describe Timber::Logger, :rails_23 => true do
         expect(io.string).to include("\"event\":{\"sql_query\":{\"sql\":\"select * from users\",\"time_ms\":56.0}}")
       end
 
-      it "should allow :time_ms" do
-        logger.info("event complete", time_ms: 54.5)
-        expect(io.string).to include("\"time_ms\":54.5")
-      end
-
       it "should allow :tag" do
         logger.info("event complete", tag: "tag1")
         expect(io.string).to include("\"tags\":[\"tag1\"]")

data/spec/timber/util/attribute_normalizer_spec.rb

@@ -0,0 +1,90 @@
+require "spec_helper"
+
+describe Timber::Util::AttributeNormalizer, :rails_23 => true do
+  describe "#fetch" do
+    it "should return nil values" do
+      normalizer = described_class.new({:key => nil})
+      v = normalizer.fetch(:key, :string)
+      expect(v).to be_nil
+    end
+
+    it "should nillify blank strings" do
+      normalizer = described_class.new({:key => ""})
+      v = normalizer.fetch(:key, :string)
+      expect(v).to be_nil
+    end
+
+    it "should nillify empty arrays" do
+      normalizer = described_class.new({:key => []})
+      v = normalizer.fetch(:key, :string)
+      expect(v).to be_nil
+    end
+
+    it "should nillify empty hashes" do
+      normalizer = described_class.new({:key => {}})
+      v = normalizer.fetch(:key, :string)
+      expect(v).to be_nil
+    end
+
+    it "should raise an error for non arrays" do
+      normalizer = described_class.new({:key => "value"})
+      expect(lambda { normalizer.fetch(:key, :array) }).to raise_error(ArgumentError)
+    end
+
+    it "should return arrays" do
+      normalizer = described_class.new({:key => [1]})
+      v = normalizer.fetch(:key, :array)
+      expect(v).to eq([1])
+    end
+
+    it "should return a float with the correct precision" do
+      normalizer = described_class.new({:key => 1.111111})
+      v = normalizer.fetch(:key, :float, :precision => 2)
+      expect(v).to eq(1.11)
+    end
+
+    it "should sanitize a hash" do
+      normalizer = described_class.new({:key => {:PASSWORD => "password"}})
+      v = normalizer.fetch(:key, :hash, :sanitize => ["password"])
+      expect(v).to eq({"password"=>"[sanitized]"})
+    end
+
+    it "should normalize encodings" do
+      value = "test".force_encoding('ASCII-8BIT')
+      normalizer = described_class.new({:key => {:key => value}})
+      v = normalizer.fetch(:key, :hash)
+      expect(v[:key].encoding).to eq(::Encoding::UTF_8)
+    end
+
+    it "should drop large binaries" do
+      value = ("a" * 1_001).force_encoding('ASCII-8BIT')
+      normalizer = described_class.new({:key => {:key => value}})
+      v = normalizer.fetch(:key, :hash)
+      expect(v).to be_nil
+    end
+
+    it "should return an integer" do
+      normalizer = described_class.new({:key => "1"})
+      v = normalizer.fetch(:key, :integer)
+      expect(v).to eq(1)
+    end
+
+    it "should limit a string" do
+      normalizer = described_class.new({:key => "aaa"})
+      v = normalizer.fetch(:key, :string, :limit => 1)
+      expect(v).to eq("a")
+    end
+
+    it "should upcase a string" do
+      normalizer = described_class.new({:key => "aaa"})
+      v = normalizer.fetch(:key, :string, :upcase => true)
+      expect(v).to eq("AAA")
+    end
+
+    it "should return a symbol" do
+      normalizer = described_class.new({:key => "sym"})
+      v = normalizer.fetch(:key, :symbol)
+      expect(v).to eq(:sym)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: timber
 version: !ruby/object:Gem::Version
-  version: 2.5.1
+  version: 2.6.0.pre.beta1
 platform: ruby
 authors:
 - Timber Technologies, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-27 00:00:00.000000000 Z
+date: 2017-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -242,9 +242,9 @@ files:
 - lib/timber/timer.rb
 - lib/timber/util.rb
 - lib/timber/util/active_support_log_subscriber.rb
+- lib/timber/util/attribute_normalizer.rb
 - lib/timber/util/hash.rb
-- lib/timber/util/http_event.rb
-- lib/timber/util/object.rb
+- lib/timber/util/non_nil_hash_builder.rb
 - lib/timber/util/request.rb
 - lib/timber/util/struct.rb
 - lib/timber/version.rb
@@ -291,7 +291,7 @@ files:
 - spec/timber/log_devices/http_spec.rb
 - spec/timber/log_entry_spec.rb
 - spec/timber/logger_spec.rb
-- spec/timber/util/http_event_spec.rb
+- spec/timber/util/attribute_normalizer_spec.rb
 - spec/timber/util/request_spec.rb
 - timber.gemspec
 homepage: https://github.com/timberio/timber-ruby
@@ -308,9 +308,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 1.9.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.5.2
@@ -361,5 +361,5 @@ test_files:
 - spec/timber/log_devices/http_spec.rb
 - spec/timber/log_entry_spec.rb
 - spec/timber/logger_spec.rb
-- spec/timber/util/http_event_spec.rb
+- spec/timber/util/attribute_normalizer_spec.rb
 - spec/timber/util/request_spec.rb

data/lib/timber/util/http_event.rb

@@ -1,69 +0,0 @@
-module Timber
-  module Util
-    # Utility module for dealing with HTTP events {Events::HTTPRequest} and {Events::HTTPResponse}.
-    module HTTPEvent
-      HEADERS_TO_SANITIZE = ['authorization', 'x-amz-security-token'].freeze
-      MAX_QUERY_STRING_BYTES = 2048.freeze
-      STRING_CLASS_NAME = 'String'.freeze
-
-      extend self
-
-      def full_path(path, query_string)
-        if query_string
-          "#{path}?#{query_string}"
-        else
-          path
-        end
-      end
-
-      # Normalizes the body. If limit if passed it will truncate the body to that limit.
-      def normalize_body(body)
-        if body.respond_to?(:body)
-          body = body.body.to_s
-        end
-
-        limit = Config.instance.http_body_limit
-        if limit
-          body.byteslice(0, limit)
-        else
-          body
-        end
-      end
-
-      # Normalizes headers to:
-      #
-      # 1. Only select values that are UTF8, otherwise they will throw errors when serializing.
-      # 2. Sanitize sensitive headers such as `Authorization` or custom headers specified in
-      def normalize_headers(headers)
-        if headers.is_a?(::Hash)
-          h = headers.each_with_object({}) do |(k, v), h|
-            if v
-              v = v.to_s
-              if [Encoding::UTF_8, Encoding::US_ASCII].include?(v.encoding)
-                h[k] = v
-              end
-            end
-          end
-
-          keys_to_sanitize = HEADERS_TO_SANITIZE + (Config.instance.http_header_filters || [])
-          Util::Hash.sanitize(h, keys_to_sanitize)
-        else
-          headers
-        end
-      end
-
-      # Normalizes the HTTP method into an uppercase string.
-      def normalize_method(method)
-        method.is_a?(::String) ? method.upcase : method
-      end
-
-      def normalize_query_string(query_string)
-        if !query_string.nil?
-          query_string = query_string.to_s
-        end
-
-        query_string && query_string.byteslice(0, MAX_QUERY_STRING_BYTES)
-      end
-    end
-  end
-end