timber 1.0.11 → 1.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d077443fc6d81d17024ca768f807d246ae550741
-  data.tar.gz: ad06e2613fa515befaf240c91ed03329f75dd432
+  metadata.gz: 66168776f4469f5a0ed9795a376b7f29fefe7a07
+  data.tar.gz: 3782864585581e766834c6a4e907b7a91aac4006
 SHA512:
-  metadata.gz: 0ee5c69b706a378a785484b71c3553d103d501b2e4ac47be9a71c53a15f9aa230f01afd6c21b7178e49c651a026fe3504687890e403c8e7c537445910f890c5e
-  data.tar.gz: aeb022f26127fdecde663867d82e1a672a9f211078d0dbd922b1fa4d4bb16f78fbdd4e503e35a9504146a6bcc7cd05a4e6872d2223cf947eef83869fe016b1fb
+  metadata.gz: c3439222a545671693a4dcf0a1124627df087e7de306775a2d682c03ca963b598f202959ea0137af793ce91ef5db3634f9d4369c859eb55196a65c0d6f7ba3b5
+  data.tar.gz: c8cecdddcc719f6b5f84453ee784e9c5b1a5d2a4b34d6d7c69868c0cf05c8789a3555de2626b6f11d52f88bc900318dc76e77926dce5046dc5f01c47b7cdd457

data/lib/timber/log_devices/http.rb

@@ -8,23 +8,15 @@ module Timber
     class HTTP
       # @private
       class LogMsgQueue
-        MAX_MSG_BYTES = 50_000 # 50kb
-
-        def initialize(max_bytes)
+        def initialize(max_size)
           @lock = Mutex.new
-          @max_bytes = max_bytes
+          @max_size = max_size
           @array = []
-          @bytesize = 0
         end
 
         def enqueue(msg)
-          if msg.bytesize > MAX_MSG_BYTES
-            raise ArgumentError.new("Log message exceeds the #{MAX_MSG_BYTES} bytes limit")
-          end
-
           @lock.synchronize do
             @array << msg
-            @bytesize += msg.bytesize
           end
         end
 
@@ -32,14 +24,13 @@ module Timber
           @lock.synchronize do
             old = @array
             @array = []
-            @bytesize = 0
             return old
           end
         end
 
         def full?
           @lock.synchronize do
-            @bytesize >= @max_bytes
+            size >= @max_size
           end
         end
 
@@ -70,7 +61,7 @@ module Timber
       end
 
       TIMBER_URL = "https://logs.timber.io/frames".freeze
-      CONTENT_TYPE = "application/x-timber-msgpack-frame-1; charset=ascii-8bit".freeze
+      CONTENT_TYPE = "application/msgpack".freeze
       USER_AGENT = "Timber Ruby Gem/#{Timber::VERSION}".freeze
       DELIVERY_FREQUENCY_SECONDS = 2.freeze
       RETRY_LIMIT = 5.freeze
@@ -90,17 +81,17 @@ module Timber
       # @param api_key [String] The API key provided to you after you add your application to
       #   [Timber](https://timber.io).
       # @param [Hash] options the options to create a HTTP log device with.
-      # @option attributes [Symbol] :batch_byte_size Determines the maximum size in bytes for
-      #   each HTTP payload. If the buffer exceeds this limit a delivery will be attempted.
-      # @option attributes [Symbol] :debug Whether to print debug output or not. This is also
+      # @option attributes [Symbol] :batch_size (500) Determines the maximum of log lines in each HTTP
+      #   payload. If the queue exceeds this limit a HTTP request will be issued.
+      # @option attributes [Symbol] :debug (false) Whether to print debug output or not. This is also
       #   inferred from ENV['debug']. Output will be sent to `Timber::Config.logger`.
       # @option attributes [Symbol] :flush_interval (2) How often the client should
       #   attempt to deliver logs to the Timber API. The HTTP client buffers logs and this
       #   options represents how often that will happen, assuming `:batch_byte_size` is not met.
-      # @option attributes [Symbol] :requests_per_conn The number of requests to send over a
+      # @option attributes [Symbol] :requests_per_conn (1000) The number of requests to send over a
       #   single persistent connection. After this number is met, the connection will be closed
       #   and a new one will be opened.
-      # @option attributes [Symbol] :request_queue The request queue object that queues Net::HTTP
+      # @option attributes [Symbol] :request_queue (SizedQueue.new(3)) The request queue object that queues Net::HTTP
       #   requests for delivery. By deafult this is a `SizedQueue` of size `3`. Meaning once
       #   3 requests are placed on the queue for delivery, back pressure will be applied. IF
       #   you'd prefer to drop messages instead, pass a {DroppingSizedQueue}. See examples for
@@ -119,12 +110,12 @@ module Timber
         @api_key = api_key
         @debug = options[:debug] || ENV['debug']
         @timber_url = URI.parse(options[:timber_url] || ENV['TIMBER_URL'] || TIMBER_URL)
-        @batch_byte_size = options[:batch_byte_size] || 3_000_000 # 3mb
+        @batch_size = options[:batch_size] || 500
         @flush_interval = options[:flush_interval] || 2 # 2 seconds
         @requests_per_conn = options[:requests_per_conn] || 1_000
-        @msg_queue = LogMsgQueue.new(@batch_byte_size)
+        @msg_queue = LogMsgQueue.new(@batch_size)
         @request_queue = options[:request_queue] || SizedQueue.new(3)
-        @req_in_flight = 0
+        @requests_in_flight = 0
 
         if options[:threads] != false
           @outlet_thread = Thread.new { outlet }
@@ -158,17 +149,12 @@ module Timber
           msgs = @msg_queue.flush
           return if msgs.empty?
 
-          body = ""
-          msgs.each do |msg|
-            body << msg
-          end
-
           req = Net::HTTP::Post.new(@timber_url.path)
           req['Accept'] = "application/json"
           req['Authorization'] = authorization_payload
           req['Content-Type'] = CONTENT_TYPE
           req['User-Agent'] = USER_AGENT
-          req.body = body
+          req.body = msgs.to_msgpack
           @request_queue.enq(req)
           @last_flush = Time.now
         end
@@ -205,9 +191,9 @@ module Timber
               http.start do |conn|
                 num_reqs = 0
                 while num_reqs < @requests_per_conn
-                  #Blocks waiting for a request.
+                  # Blocks waiting for a request.
                   req = @request_queue.deq
-                  @req_in_flight += 1
+                  @requests_in_flight += 1
                   resp = nil
                   begin
                     resp = conn.request(req)
@@ -215,7 +201,7 @@ module Timber
                     logger.error("Timber request error: #{e.message}") if debug?
                     next
                   ensure
-                    @req_in_flight -= 1
+                    @requests_in_flight -= 1
                   end
                   num_reqs += 1
                   logger.info("Timber request successful: #{resp.code}") if debug?

data/lib/timber/log_entry.rb

@@ -51,6 +51,10 @@ module Timber
       as_json(options).to_json
     end
 
+    def to_msgpack(*args)
+      as_json.to_msgpack(*args)
+    end
+
     private
       def formatted_dt
         @formatted_dt ||= time.iso8601(DT_PRECISION)

data/lib/timber/logger.rb

@@ -119,31 +119,14 @@ module Timber
       end
     end
 
-    # Structures your log messages into JSON.
-    #
-    #   logger = Timber::Logger.new(STDOUT)
-    #   logger.formatter = Timber::JSONFormatter.new
-    #
-    # Example message:
-    #
-    #   {"level":"info","dt":"2016-09-01T07:00:00.000000-05:00","message":"My log message"}
-    #
-    class MsgPackFormatter < Formatter
+    # Passes through the LogEntry object. This is specifically used for the {Timber::LogDevices::HTTP}
+    # class. This allows the IO device to format it however it wants. This is neccessary for
+    # MessagePack because it requires a fixed array size before encoding. And since HTTP is
+    # sending data in batches, the encoding should happen there.
+    class PassThroughFormatter < Formatter
       def call(severity, time, progname, msg)
-        # use << for concatenation for performance reasons
-        hash = build_log_entry(severity, time, progname, msg).as_json
-        to_msgpack(hash) << "\n"
+        build_log_entry(severity, time, progname, msg)
       end
-
-      private
-        def to_msgpack(msg)
-          begin
-            msg.to_msgpack
-          rescue NoMethodError
-            json = JSON.generate(msg)
-            JSON.parse(json).to_msgpack
-          end
-        end
     end
 
     # Creates a new Timber::Logger instances. Accepts the same arguments as `::Logger.new`.
@@ -156,7 +139,7 @@ module Timber
     def initialize(*args)
       super(*args)
       if args.size == 1 and args.first.is_a?(LogDevices::HTTP)
-        self.formatter = MsgPackFormatter.new
+        self.formatter = PassThroughFormatter.new
       else
         self.formatter = HybridFormatter.new
       end
@@ -165,7 +148,7 @@ module Timber
     def formatter=(value)
       if @dev.is_a?(Timber::LogDevices::HTTP)
         raise ArgumentError.new("The formatter cannot be changed when using the " +
-          "Timber::LogDevices::HTTP log device. The MsgPackFormatter must be used for proper " +
+          "Timber::LogDevices::HTTP log device. The PassThroughFormatter must be used for proper " +
           "delivery.")
       end
       super

data/lib/timber/version.rb

@@ -1,3 +1,3 @@
 module Timber
-  VERSION = "1.0.11"
+  VERSION = "1.0.12"
 end

data/spec/timber/log_devices/http_spec.rb

@@ -20,12 +20,11 @@ describe Timber::LogDevices::HTTP do
       expect(msg_queue.flush).to eq(["test log message"])
     end
 
-    context "with a low batch byte size" do
-      let(:http) { described_class.new("MYKEY", :batch_byte_size => 20) }
+    context "with a low batch size" do
+      let(:http) { described_class.new("MYKEY", :batch_size => 2) }
 
-      it "should attempt a delivery when the payload limit is exceeded" do
-        message = "a" * 19
-        http.write(message)
+      it "should attempt a delivery when the limit is exceeded" do
+        http.write("test")
         expect(http).to receive(:flush).exactly(1).times
         http.write("my log message")
       end
@@ -55,18 +54,30 @@ describe Timber::LogDevices::HTTP do
 
   # Testing a private method because it helps break down our tests
   describe "#flush" do
+    let(:time) { Time.utc(2016, 9, 1, 12, 0, 0) }
+
     it "should add a request to the queue" do
       http = described_class.new("MYKEY", threads: false)
-      http.write("This is a log message")
+      log_entry = Timber::LogEntry.new("INFO", time, nil, "test log message 1", nil, nil)
+      http.write(log_entry)
+      log_entry = Timber::LogEntry.new("INFO", time, nil, "test log message 2", nil, nil)
+      http.write(log_entry)
       http.send(:flush)
       request_queue = http.instance_variable_get(:@request_queue)
       request = request_queue.deq
       expect(request).to be_kind_of(Net::HTTP::Post)
-      expect(request.body).to eq("This is a log message")
+      expect(request.body).to eq("\x92\x83\xA5level\xA4INFO\xA2dt\xBB2016-09-01T12:00:00.000000Z\xA7message\xB2test log message 1\x83\xA5level\xA4INFO\xA2dt\xBB2016-09-01T12:00:00.000000Z\xA7message\xB2test log message 2".force_encoding("ASCII-8BIT"))
 
       message_queue = http.instance_variable_get(:@msg_queue)
       expect(message_queue.size).to eq(0)
     end
+
+    it "should preserve formatting for mshpack payloads" do
+      http = described_class.new("MYKEY", threads: false)
+      http.write("This is a log message 1".to_msgpack)
+      http.write("This is a log message 2".to_msgpack)
+      http.send(:flush)
+    end
   end
 
   # Testing a private method because it helps break down our tests
@@ -74,30 +85,34 @@ describe Timber::LogDevices::HTTP do
     it "should start a intervaled flush thread and flush on an interval" do
       http = described_class.new("MYKEY", flush_interval: 0.1)
       expect(http).to receive(:flush).exactly(1).times
-      sleep 0.15 # too fast!
+      sleep 0.12 # too fast!
       mock = expect(http).to receive(:flush).exactly(1).times
-      sleep 0.15 # too fast!
+      sleep 0.12 # too fast!
     end
   end
 
   # Outlet
   describe "#outlet" do
+    let(:time) { Time.utc(2016, 9, 1, 12, 0, 0) }
+
     it "should start a intervaled flush thread and flush on an interval" do
       stub = stub_request(:post, "https://logs.timber.io/frames").
         with(
-          :body => "test log message 1test log message 2",
+          :body => "\x92\x83\xA5level\xA4INFO\xA2dt\xBB2016-09-01T12:00:00.000000Z\xA7message\xB2test log message 1\x83\xA5level\xA4INFO\xA2dt\xBB2016-09-01T12:00:00.000000Z\xA7message\xB2test log message 2".force_encoding("ASCII-8BIT"),
           :headers => {
             'Accept' => 'application/json',
             'Authorization' => 'Basic TVlLRVk=',
-            'Content-Type' => 'application/x-timber-msgpack-frame-1; charset=ascii-8bit',
+            'Content-Type' => 'application/msgpack',
             'User-Agent' => "Timber Ruby Gem/#{Timber::VERSION}"
           }
         ).
         to_return(:status => 200, :body => "", :headers => {})
 
       http = described_class.new("MYKEY", flush_interval: 0.1)
-      http.write("test log message 1")
-      http.write("test log message 2")
+      log_entry = Timber::LogEntry.new("INFO", time, nil, "test log message 1", nil, nil)
+      http.write(log_entry)
+      log_entry = Timber::LogEntry.new("INFO", time, nil, "test log message 2", nil, nil)
+      http.write(log_entry)
       sleep 0.3
 
       expect(stub).to have_been_requested.times(1)

data/spec/timber/logger_spec.rb

@@ -93,32 +93,8 @@ describe Timber::Logger, :rails_23 => true do
     context "with the HTTP log device" do
       let(:io) { Timber::LogDevices::HTTP.new("my_key") }
 
-      it "should use the msgpack formatter" do
-        expect(logger.formatter).to be_kind_of(Timber::Logger::MsgPackFormatter)
-      end
-
-      it "should log properly with the msgpack format" do
-        event = Timber::Events::Custom.new(
-          type: :payment_rejected,
-          message: "Payment rejected",
-          data: {customer_id: "abcd1234", amount: 100}
-        )
-        expect(io).to receive(:write).exactly(1).times
-        logger.error(event)
-      end
-    end
-  end
-
-  describe described_class::MsgPackFormatter do
-    describe "#call" do
-      let(:time) { Time.utc(2016, 9, 1, 12, 0, 0) }
-
-      it "should produce a message pack formatted message" do
-        formatter = described_class.new
-        result = formatter.call("INFO", time, nil, "this is a test message")
-        hash = {"level"=>"info", "dt"=>"2016-09-01T12:00:00.000000Z", "message"=>"this is a test message"}
-        expected = hash.to_msgpack + "\n"
-        expect(result).to eq(expected)
+      it "should use the PassThroughFormatter" do
+        expect(logger.formatter).to be_kind_of(Timber::Logger::PassThroughFormatter)
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: timber
 version: !ruby/object:Gem::Version
-  version: 1.0.11
+  version: 1.0.12
 platform: ruby
 authors:
 - Timber Technologies, Inc.