timber 2.1.10 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e83f2fd0fc9e2e55f2a81089ac80ef335255b82f
-  data.tar.gz: f860cc79667cc15469caf10600abfd9f4c5281a6
+  metadata.gz: d36593e9185855c949d8675ac749043317998c3c
+  data.tar.gz: 479e0355abf6e918aa1d0e0566e152c5fcfdaf97
 SHA512:
-  metadata.gz: 81505965ac76980955b226ec83f5f43ba166b1ab671b5175a2a9f8f68e973bd7c320355c719f9b03792debe81ceac25c2e2832a517b1b931b31c28e8e31afb23
-  data.tar.gz: b2a072f8ca6c6092c3e2ecff9040ca420b1fb32a0282b728f81d5bc90688619bb06e1062c2c11834bb10f91bd182c3d7755ecebdd591850eb0dd5ce04bb5c7b8
+  metadata.gz: 70397717e17995dcbf20b2d667037307a3f19c9702325442ea896e2d0c7c424adb4eea1bb7bba4e7b70d511acf11235018b639e2b7aef72b800e11965f6d0f8c
+  data.tar.gz: b4f322a1167c530eb1b5e0a90fc836781619a93d0f7a2e05ce4bcbb342bb53ef2356195c8df8ea5055f75cbeac63c19fb7598b891d6a87c8210e7c510f3b8378

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
-Please see [https://github.com/timberio/timber-ruby/releases](https://github.com/timberio/timber-ruby/releases) for library specific changes.
+All notable changes to this project will be documented in this file.
 
-For all Timber changes see [https://timber.io/changelog](https://timber.io/changelog).
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [2.2.0] - 2017-09-13
+### Changed
+
+  - The default HTTP log device queue type was switched to a
+    `Timber::LogDevices::HTTP::FlushableDroppingSizedQueue` instead of a `::SizedQueue`. In the
+    event of extremely high volume logging, and delivery cannot keep up, Timber will drop messages
+    instead of applying back pressure.
+
+
+[Unreleased]: https://github.com/timberio/timber-ruby/compare/v2.2.0...HEAD
+[2.2.0]: https://github.com/timberio/timber-ruby/compare/v2.1.10...v2.2.0

data/lib/timber/log_devices/http.rb

@@ -3,8 +3,8 @@ require "msgpack"
 require "net/https"
 
 require "timber/config"
-require "timber/log_devices/http/dropping_sized_queue"
-require "timber/log_devices/http/flushable_sized_queue"
+require "timber/log_devices/http/flushable_dropping_sized_queue"
+require "timber/log_devices/http/request_attempt"
 require "timber/version"
 
 module Timber
@@ -52,10 +52,11 @@ module Timber
       # @option attributes [Symbol] :requests_per_conn (2500) The number of requests to send over a
       #   single persistent connection. After this number is met, the connection will be closed
       #   and a new one will be opened.
-      # @option attributes [Symbol] :request_queue (SizedQueue.new(3)) The request queue object that queues Net::HTTP
-      #   requests for delivery. By deafult this is a `SizedQueue` of size `3`. Meaning once
-      #   3 requests are placed on the queue for delivery, back pressure will be applied. IF
-      #   you'd prefer to drop messages instead, pass a {DroppingSizedQueue}. See examples for
+      # @option attributes [Symbol] :request_queue (FlushableDroppingSizedQueue.new(25)) The request
+      #   queue object that queues Net::HTTP requests for delivery. By deafult this is a
+      #   `FlushableDroppingSizedQueue` of size `25`. Meaning once the queue fills up to 25
+      #   requests new requests will be dropped. If you'd prefer to apply back pressure,
+      #   ensuring you do not lose log data, pass a standard {SizedQueue}. See examples for
       #   an example.
       # @option attributes [Symbol] :timber_url The Timber URL to delivery the log lines. The
       #   default is set via {TIMBER_URL}.
@@ -63,19 +64,18 @@ module Timber
       # @example Basic usage
       #   Timber::Logger.new(Timber::LogDevices::HTTP.new("my_timber_api_key"))
       #
-      # @example Dropping messages instead of applying back pressure
-      #   http_log_device = Timber::LogDevices::HTTP.new("my_timber_api_key",
-      #     request_queue: Timber::LogDevices::HTTP::DroppingSizedQueue.new(3))
+      # @example Apply back pressure instead of dropping messages
+      #   http_log_device = Timber::LogDevices::HTTP.new("my_timber_api_key", request_queue: SizedQueue.new(25))
       #   Timber::Logger.new(http_log_device)
       def initialize(api_key, options = {})
         @api_key = api_key || raise(ArgumentError.new("The api_key parameter cannot be blank"))
         @timber_url = URI.parse(options[:timber_url] || ENV['TIMBER_URL'] || TIMBER_URL)
         @batch_size = options[:batch_size] || 1_000
         @flush_continuously = options[:flush_continuously] != false
-        @flush_interval = options[:flush_interval] || 1 # 1 second
+        @flush_interval = options[:flush_interval] || 2 # 2 seconds
         @requests_per_conn = options[:requests_per_conn] || 2_500
-        @msg_queue = FlushableSizedQueue.new(@batch_size)
-        @request_queue = options[:request_queue] || SizedQueue.new(3)
+        @msg_queue = FlushableDroppingSizedQueue.new(@batch_size)
+        @request_queue = options[:request_queue] || FlushableDroppingSizedQueue.new(25)
         @successive_error_count = 0
         @requests_in_flight = 0
       end
@@ -85,7 +85,7 @@ module Timber
       # size is constricted by the Timber API. The actual application limit is a multiple
       # of this. Hence the `@request_queue`.
       def write(msg)
-        @msg_queue.enqueue(msg)
+        @msg_queue.enq(msg)
 
         # Lazily start flush threads to ensure threads are alive after forking processes.
         # If the threads are started during instantiation they will not be copied when
@@ -161,7 +161,8 @@ module Timber
           req = build_request
           if !req.nil?
             Timber::Config.instance.debug { "New request placed on queue" }
-            @request_queue.enq(req)
+            request_attempt = RequestAttempt.new(req)
+            @request_queue.enq(request_attempt)
           end
         end
 
@@ -255,36 +256,39 @@ module Timber
           while num_reqs < @requests_per_conn
             Timber::Config.instance.debug { "Waiting on next request, threads waiting: #{@request_queue.num_waiting}" }
 
-            # Blocks waiting for a request.
-            req = @request_queue.deq
-            @requests_in_flight += 1
-
-            begin
-              resp = conn.request(req)
-            rescue => e
-              Timber::Config.instance.debug { "#deliver_request error: #{e.message}" }
-
-              @successive_error_count += 1
-
-              # Back off so that we don't hammer the Timber API.
-              calculated_backoff = @successive_error_count * 2
-              backoff = calculated_backoff > 30 ? 30 : calculated_backoff
-
-              Timber::Config.instance.debug { "Backing off #{backoff} seconds, error ##{@successive_error_count}" }
-
-              sleep backoff
+            request_attempt = @request_queue.deq
+
+            if request_attempt.nil?
+              sleep(1)
+            else
+              request_attempt.attempted!
+              @requests_in_flight += 1
+
+              begin
+                resp = conn.request(request_attempt.request)
+              rescue => e
+                Timber::Config.instance.debug { "#deliver_requests error: #{e.message}" }
+
+                # Throw the request back on the queue for a retry if it has been attempted less
+                # than 3 times
+                if request_attempt.attempts < 3
+                  Timber::Config.instance.debug { "Request is being retried, #{request_attempt.attempts} previous attempts" }
+                  @request_queue.enq(request_attempt)
+                else
+                  Timber::Config.instance.debug { "Request is being dropped, #{request_attempt.attempts} previous attempts" }
+                end
+
+                return false
+              ensure
+                @requests_in_flight -= 1
+              end
 
-              # Throw the request back on the queue for a retry
-              @request_queue.enq(req)
-              return false
-            ensure
-              @requests_in_flight -= 1
+              num_reqs += 1
+              Timber::Config.instance.debug { "Request successful: #{resp.code}" }
             end
-
-            @successive_error_count = 0
-            num_reqs += 1
-            Timber::Config.instance.debug { "Request successful: #{resp.code}" }
           end
+
+          true
         end
 
         # Builds the `Authorization` header value for HTTP delivery to the Timber API.

data/lib/timber/log_devices/http/{flushable_sized_queue.rb → flushable_dropping_sized_queue.rb}

@@ -2,11 +2,12 @@ module Timber
   module LogDevices
     class HTTP
       # A simple thread-safe queue implementation that provides a #flush method.
-      # The built-in ruby Queue class does not provide a #flush method. It also
-      # implement thread waiting which is something we do not want. To keep things
-      # simple and straight-forward we designed our own simple queue class.
+      # The built-in ruby `Queue` class does not provide a #flush method that allows
+      # the caller to retrieve all items on the queue in one call. The Ruby `SizedQueue` also
+      # implements thread waiting, which is something we want to avoid. To keep things
+      # simple and straight-forward, we designed this queue class.
       # @private
-      class FlushableSizedQueue
+      class FlushableDroppingSizedQueue
         def initialize(max_size)
           @lock = Mutex.new
           @max_size = max_size
@@ -14,9 +15,18 @@ module Timber
         end
 
         # Adds a message to the queue
-        def enqueue(msg)
+        def enq(msg)
           @lock.synchronize do
-            @array << msg
+            if !full?
+              @array << msg
+            end
+          end
+        end
+
+        # Removes a single item from the queue
+        def deq
+          @lock.synchronize do
+            @array.pop
           end
         end
 

data/lib/timber/log_devices/http/request_attempt.rb

@@ -0,0 +1,20 @@
+module Timber
+  module LogDevices
+    class HTTP
+      # Represents an attempt to deliver a request. Requests can be retried, hence
+      # why we keep track of the number of attempts.
+      class RequestAttempt
+        attr_reader :attempts, :request
+
+        def initialize(req)
+          @attempts = 0
+          @request = req
+        end
+
+        def attempted!
+          @attempts += 1
+        end
+      end
+    end
+  end
+end

data/lib/timber/version.rb

@@ -1,3 +1,3 @@
 module Timber
-  VERSION = "2.1.10"
+  VERSION = "2.2.0"
 end

data/spec/timber/log_devices/http_spec.rb

@@ -100,9 +100,9 @@ describe Timber::LogDevices::HTTP do
       http.write(log_entry)
       http.send(:flush_async)
       request_queue = http.instance_variable_get(:@request_queue)
-      request = request_queue.deq
-      expect(request).to be_kind_of(Net::HTTP::Post)
-      expect(request.body).to start_with("\x92\x84\xA5level\xA4INFO\xA2dt\xBB2016-09-01T12:00:00.000000Z\xA7message\xB2test log message 1".force_encoding("ASCII-8BIT"))
+      request_attempt = request_queue.deq
+      expect(request_attempt.request).to be_kind_of(Net::HTTP::Post)
+      expect(request_attempt.request.body).to start_with("\x92\x84\xA5level\xA4INFO\xA2dt\xBB2016-09-01T12:00:00.000000Z\xA7message\xB2test log message 1".force_encoding("ASCII-8BIT"))
 
       message_queue = http.instance_variable_get(:@msg_queue)
       expect(message_queue.size).to eq(0)
@@ -141,7 +141,7 @@ describe Timber::LogDevices::HTTP do
       http.write(log_entry1)
       log_entry2 = Timber::LogEntry.new("INFO", time, nil, "test log message 2", nil, nil)
       http.write(log_entry2)
-      sleep 1
+      sleep 2
 
       expect(stub).to have_been_requested.times(1)
 
@@ -157,8 +157,10 @@ describe Timber::LogDevices::HTTP do
       req_queue = http_device.instance_variable_get(:@request_queue)
 
       # Place a request on the queue
-      req = Net::HTTP::Post.new("/")
-      req_queue.enq(req)
+      request = Net::HTTP::Post.new("/")
+      request_attempt = Timber::LogDevices::HTTP::RequestAttempt.new(request)
+      request_attempt.attempted!
+      req_queue.enq(request_attempt)
 
       # Start a HTTP connection to test the method directly
       http = http_device.send(:build_http)
@@ -168,6 +170,16 @@ describe Timber::LogDevices::HTTP do
       end
 
       expect(req_queue.size).to eq(1)
+
+      # Start a HTTP connection to test the method directly
+      http = http_device.send(:build_http)
+      http.start do |conn|
+        result = http_device.send(:deliver_requests, conn)
+        expect(result).to eq(false)
+      end
+
+      # Ensure the request gets discards after 3 attempts
+      expect(req_queue.size).to eq(0)
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: timber
 version: !ruby/object:Gem::Version
-  version: 2.1.10
+  version: 2.2.0
 platform: ruby
 authors:
 - Timber Technologies, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-29 00:00:00.000000000 Z
+date: 2017-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -229,8 +229,8 @@ files:
 - lib/timber/integrator.rb
 - lib/timber/log_devices.rb
 - lib/timber/log_devices/http.rb
-- lib/timber/log_devices/http/dropping_sized_queue.rb
-- lib/timber/log_devices/http/flushable_sized_queue.rb
+- lib/timber/log_devices/http/flushable_dropping_sized_queue.rb
+- lib/timber/log_devices/http/request_attempt.rb
 - lib/timber/log_entry.rb
 - lib/timber/logger.rb
 - lib/timber/overrides.rb

data/lib/timber/log_devices/http/dropping_sized_queue.rb

@@ -1,26 +0,0 @@
-module Timber
-  module LogDevices
-    class HTTP
-      # Works like SizedQueue, but drops message instead of blocking. Pass one of these in
-      # to {HTTP#intiialize} via the :request_queue option if you'd prefer to drop messages
-      # in the event of a buffer overflow instead of applying back pressure.
-      class DroppingSizedQueue < SizedQueue
-        # Returns true/false depending on whether the queue is full or not
-        def push(obj)
-          @mutex.synchronize do
-            return false unless @que.length < @max
-
-            @que.push obj
-            begin
-              t = @waiting.shift
-              t.wakeup if t
-            rescue ThreadError
-              retry
-            end
-            return true
-          end
-        end
-      end
-    end
-  end
-end