timber 2.0.20 → 2.0.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 946dc64576210d225bbf4952fa78363705b3b87f
-  data.tar.gz: e972b58acf8efc60ebcc79746939d7bbceeadd06
+  metadata.gz: f65e44d5582af9d177dabd02f2229adda5ea094b
+  data.tar.gz: a113bb27776a6403b3b237b49d33c7fbac962758
 SHA512:
-  metadata.gz: f44fdf7da119df5a80cd1cb589f1c55d8fee39bfccd6cacf35eaf9b5944afdc227cb6d715473c651d3830035915bc62df11cae4c7c58a779331870fa23659d26
-  data.tar.gz: 7cad85ce7ea3c9df75fd2d3fc7e18df07d42d20f294bba6af8803dce710f7b9a7e0a8990ef243d3feff942807625ffb8b40bc2ecde89c98e2604a7e129c94533
+  metadata.gz: 10b371f99e2c4f4203526917a770140095581e17a12b72fba44dfc8db8b2151a198b5e771527fb715f6c9c4fe9e0d1c56772ff6663fb31c07f76bd95210339d4
+  data.tar.gz: e0b9fd2b91d7de66b4bee5305337afeb5017ec42fff2c76e8374c957af5d88445701aa029fca846f2f7e7778e4ffa7231989ea6c05d3814f307e244e5b2607d0

data/.travis.yml

@@ -22,7 +22,7 @@ env:
 before_script:
 - echo $BUNDLE_GEMFILE
 - bundle install
-script: bundle exec rspec
+script: bundle exec rspec --format documentation
 matrix:
   fast_finish: true
   exclude:

data/lib/timber/config.rb

@@ -1,3 +1,4 @@
+require "logger"
 require "singleton"
 
 module Timber
@@ -13,6 +14,13 @@ module Timber
   class Config
     class NoLoggerError < StandardError; end
 
+    class SimpleFormatter < ::Logger::Formatter
+      # This method is invoked when a log event occurs
+      def call(severity, timestamp, progname, msg)
+        "[Timber] #{String === msg ? msg : msg.inspect}\n"
+      end
+    end
+
     PRODUCTION_NAME = "production".freeze
     STAGING_NAME = "staging".freeze
 
@@ -54,6 +62,7 @@ module Timber
       file.binmode
       file.sync = config.autoflush_log
       file_logger = ::Logger.new(file)
+      file_logger.formatter = SimpleFormatter.new
       self.debug_logger = file_logger
     end
 
@@ -65,6 +74,7 @@ module Timber
     #   Timber::Config.instance.debug_to_file("log/timber.log")
     def debug_to_stdout
       stdout_logger = ::Logger.new(STDOUT)
+      stdout_logger.formatter = SimpleFormatter.new
       self.debug_logger = stdout_logger
     end
 
@@ -133,7 +143,11 @@ module Timber
     # @example Everything else
     #   Timber::Config.instance.logger = Timber::Logger.new(STDOUT)
     def logger
-      @logger || Logger.new(STDOUT)
+      if @logger.is_a?(Proc)
+        @logger.call()
+      else
+        @logger ||= Logger.new(STDOUT)
+      end
     end
 
     private

data/lib/timber/frameworks.rb

@@ -3,12 +3,12 @@ require "logger"
 # Attempt to require Rails. We can not list it as a gem
 # dependency because we want to support multiple frameworks.
 begin
-  require("rails")
+  require "rails"
 rescue LoadError
 end
 
 if defined?(::Rails) && defined?(::Rails::Railtie)
-  require 'timber/frameworks/rails'
+  require "timber/frameworks/rails"
 end
 
 module Timber

data/lib/timber/frameworks/rails.rb

@@ -48,14 +48,17 @@ module Timber
       class Railtie < ::Rails::Railtie
         config.timber = Config.instance
 
-        # Initialize Timber immediately after the logger in case anything uses the logger
-        # during the initialization process.
-        initializer(:timber, after: :initialize_logger) do
-          logger = Rails.ensure_timber_logger(::Rails.logger)
-          Rails.set_logger(logger)
+        config.before_initialize do
+          Timber::Config.instance.logger = Proc.new { ::Rails.logger }
+        end
 
+        # Must be loaded after initializers so that we respect any Timber configuration
+        # set
+        initializer(:timber, after: :load_config_initializers) do
           Integrations.integrate!
 
+          # Install the Rack middlewares so that we capture structured data instead of
+          # raw text logs.
           timber_operations = Integrations::Rack.middlewares.collect do |middleware_class|
             [:use, [middleware_class], nil]
           end
@@ -63,43 +66,6 @@ module Timber
           config.app_middleware.timber_operations = timber_operations
         end
       end
-
-      # This builds a new Timber::Logger from an existing logger. This allows us to transparentl
-      # switch users onto the Timber::Logger since we support a more useful logging API.
-      def self.ensure_timber_logger(existing_logger)
-        if existing_logger.is_a?(Logger)
-          return existing_logger
-        end
-
-        log_device = existing_logger.instance_variable_get(:@logdev).try(:dev)
-        logger = Logger.new(log_device)
-        logger.level = existing_logger.try(:level) || Logger::DEBUG
-        if defined?(::ActiveSupport::TaggedLogging)
-          logger = ::ActiveSupport::TaggedLogging.new(logger)
-        end
-        logger
-      end
-
-      # Sets the Rails logger. Rails
-      def self.set_logger(logger)
-        if defined?(::ActiveSupport::TaggedLogging) && !logger.is_a?(::ActiveSupport::TaggedLogging)
-          logger = ::ActiveSupport::TaggedLogging.new(logger)
-        end
-
-        Config.instance.logger = logger
-
-        # Set the various Rails framework loggers. We *have* to do this because Rails
-        # internally sets these with an ActiveSupport.onload(:active_record) { } callback.
-        # We don't have an opportunity to intercept this since the :initialize_logger
-        # initializer loads these modules. Moreover, earlier version of rails don't do this
-        # at all, hence the defined? checks. Yay for being implicit.
-        ::ActionCable::Server::Base.logger = logger if defined?(::ActionCable::Server::Base) && ::ActionCable::Server::Base.respond_to?(:logger=)
-        ::ActionController::Base.logger = logger if defined?(::ActionController::Base) && ::ActionController::Base.respond_to?(:logger=)
-        ::ActionMailer::Base.logger = logger if defined?(::ActionMailer::Base) && ::ActionMailer::Base.respond_to?(:logger=)
-        ::ActionView::Base.logger = logger if defined?(::ActionView::Base) && ::ActionView::Base.respond_to?(:logger=)
-        ::ActiveRecord::Base.logger = logger if defined?(::ActiveRecord::Base) && ::ActiveRecord::Base.respond_to?(:logger=)
-        ::Rails.logger = logger
-      end
     end
   end
 end

data/lib/timber/integrations/active_record/log_subscriber/timber_log_subscriber.rb

@@ -1,6 +1,6 @@
-# We require all of ActiveRecord because #logger usses ActiveRecord::Base.
-# We can't require active_record/base directly because ActiveRecord does not require
-# files properly.
+# We require all of ActiveRecord because the #logger method in ::ActiveRecord::LogSubscriber
+# uses ActiveRecord::Base. We can't require active_record/base directly because ActiveRecord
+# does not require files properly and we receive unintialized constant errors.
 require "active_record"
 require "active_record/log_subscriber"
 

data/lib/timber/integrator.rb

@@ -6,7 +6,18 @@ module Timber
     class RequirementNotMetError < StandardError; end
 
     class << self
+      attr_writer :enabled
+
+      def enabled?
+        @enabled != false
+      end
+
       def integrate!(*args)
+        if !enabled?
+          Config.instance.debug_logger.debug("#{name} integration disabled, skipping") if Config.instance.debug_logger
+          return false
+        end
+
         new(*args).integrate!
         Config.instance.debug_logger.debug("Integrated #{name}") if Config.instance.debug_logger
         true

data/lib/timber/log_devices/http.rb

@@ -5,7 +5,8 @@ module Timber
   module LogDevices
     # A highly efficient log device that buffers and delivers log messages over HTTPS to
     # the Timber API. It uses batches, keep-alive connections, and msgpack to deliver logs with
-    # high-throughput and little overhead.
+    # high-throughput and little overhead. All log preparation and delivery is done asynchronously
+    # in a thread as not to block application execution.
     #
     # See {#initialize} for options and more details.
     class HTTP
@@ -118,6 +119,7 @@ module Timber
         @requests_per_conn = options[:requests_per_conn] || 2_500
         @msg_queue = LogMsgQueue.new(@batch_size)
         @request_queue = options[:request_queue] || SizedQueue.new(3)
+        @successive_error_count = 0
         @requests_in_flight = 0
       end
 
@@ -133,7 +135,7 @@ module Timber
         ensure_flush_threads_are_started
 
         if @msg_queue.full?
-          debug_logger.debug("Flushing timber buffer via write") if debug_logger
+          debug_logger.debug("Flushing HTTP buffer via write") if debug_logger
           flush
         end
         true
@@ -154,9 +156,26 @@ module Timber
 
       # Closes the log device, cleans up, and attempts one last delivery.
       def close
+        # Kill the flush thread immediately since we are about to flush again.
         @flush_thread.kill if @flush_thread
-        @outlet_thread.kill if @outlet_thread
+
+        # Flush all remaining messages
         flush
+
+        # Kill the request_outlet thread gracefully. We do not want to kill it while a
+        # request is inflight. Ideally we'd let it finish before we die.
+        if @request_outlet_thread
+          4.times do
+            if @requests_in_flight == 0 && @request_queue.size == 0
+              @request_outlet_thread.kill
+              break
+            else
+              debug_logger.error("Busy delivering the final log messages, " +
+                "connection will close when complete.")
+              sleep 1
+            end
+          end
+        end
       end
 
       private
@@ -170,8 +189,8 @@ module Timber
         # threads are started after process forking.
         def ensure_flush_threads_are_started
           if @flush_continuously
-            if @outlet_thread.nil? || !@outlet_thread.alive?
-              @outlet_thread = Thread.new { outlet }
+            if @request_outlet_thread.nil? || !@request_outlet_thread.alive?
+              @request_outlet_thread = Thread.new { request_outlet }
             end
 
             if @flush_thread.nil? || !@flush_thread.alive?
@@ -183,15 +202,17 @@ module Timber
         def intervaled_flush
           # Wait specified time period before starting
           sleep @flush_interval
+
           loop do
             begin
               if intervaled_flush_ready?
-                debug_logger.debug("Flushing timber buffer via the interval") if debug_logger
+                debug_logger.debug("Flushing HTTP buffer via the interval") if debug_logger
                 flush
               end
-              sleep(0.1)
+
+              sleep(0.5)
             rescue Exception => e
-              logger.error("Timber intervaled flush failed: #{e.inspect}\n\n#{e.backtrace}")
+              logger.error("Intervaled HTTP flush failed: #{e.inspect}\n\n#{e.backtrace}")
             end
           end
         end
@@ -200,50 +221,73 @@ module Timber
           @last_flush.nil? || (Time.now.to_f - @last_flush.to_f).abs >= @flush_interval
         end
 
-        def outlet
+        def build_http
+          http = Net::HTTP.new(@timber_url.host, @timber_url.port)
+          http.set_debug_output(debug_logger) if debug_logger
+          http.use_ssl = true if @timber_url.scheme == 'https'
+          http.read_timeout = 30
+          http.ssl_timeout = 10
+          http.open_timeout = 10
+          http
+        end
+
+        def request_outlet
           loop do
-            http = Net::HTTP.new(@timber_url.host, @timber_url.port)
-            http.set_debug_output(debug_logger) if debug_logger
-            http.use_ssl = true if @timber_url.scheme == 'https'
-            http.read_timeout = 30
-            http.ssl_timeout = 10
-            http.open_timeout = 10
+            http = build_http
 
             begin
-              debug_logger.info("Starting Timber HTTP connection") if debug_logger
+              debug_logger.info("Starting HTTP connection") if debug_logger
+
               http.start do |conn|
-                num_reqs = 0
-                while num_reqs < @requests_per_conn
-                  if debug_logger
-                    debug_logger.debug("Waiting on next Timber request")
-                    debug_logger.debug("Number of threads waiting on Timber request queue: #{@request_queue.num_waiting}")
-                  end
-
-                  # Blocks waiting for a request.
-                  req = @request_queue.deq
-                  @requests_in_flight += 1
-                  resp = nil
-                  begin
-                    resp = conn.request(req)
-                  rescue => e
-                    debug_logger.error("Timber request error: #{e.message}") if debug_logger
-                    next
-                  ensure
-                    @requests_in_flight -= 1
-                  end
-                  num_reqs += 1
-                  debug_logger.debug("Timber request successful: #{resp.code}") if debug_logger
-                end
+                deliver_requests(conn)
               end
             rescue => e
-              debug_logger.error("Timber request error: #{e.message}") if debug_logger
+              debug_logger.error("#request_outlet error: #{e.message}") if debug_logger
             ensure
-              debug_logger.debug("Finishing Timber HTTP connection") if debug_logger
+              debug_logger.info("Finishing HTTP connection") if debug_logger
               http.finish if http.started?
             end
           end
         end
 
+        def deliver_requests(conn)
+          num_reqs = 0
+
+          while num_reqs < @requests_per_conn
+            debug_logger.info("Waiting on next request, threads waiting: #{@request_queue.num_waiting}") if debug_logger
+
+            # Blocks waiting for a request.
+            req = @request_queue.deq
+            @requests_in_flight += 1
+
+            begin
+              resp = conn.request(req)
+            rescue => e
+              debug_logger.error("#deliver_request error: #{e.message}") if debug_logger
+
+              @successive_error_count += 1
+
+              # Back off so that we don't hammer the Timber API.
+              calculated_backoff = @successive_error_count * 2
+              backoff = calculated_backoff > 30 ? 30 : calculated_backoff
+
+              debug_logger.error("Backing off #{backoff} seconds, error ##{@successive_error_count}") if debug_logger
+
+              sleep backoff
+
+              # Throw the request back on the queue for a retry
+              @request_queue.enq(req)
+              return false
+            ensure
+              @requests_in_flight -= 1
+            end
+
+            @successive_error_count = 0
+            num_reqs += 1
+            debug_logger.info("Request successful: #{resp.code}") if debug_logger
+          end
+        end
+
         def authorization_payload
           @authorization_payload ||= "Basic #{Base64.urlsafe_encode64(@api_key).chomp}"
         end

data/lib/timber/log_entry.rb

@@ -71,6 +71,10 @@ module Timber
       Util::Hash.deep_compact(hash)
     end
 
+    def inspect
+      to_s
+    end
+
     def to_json(options = {})
       as_json(options).to_json
     end
@@ -79,6 +83,26 @@ module Timber
       as_json.to_msgpack(*args)
     end
 
+    # This is used when LogEntry objects make it to a non-Timber logger.
+    def to_s
+      log_message = message
+
+      if !event.nil?
+        event_hash = event.as_json
+        event_type = event_hash.keys.first
+
+        event_type = if event.is_a?(Events::Custom)
+          "event:#{event_type}.#{event.type}"
+        else
+          "event:#{event_type}"
+        end
+
+        log_message = "#{message} [#{event_type}]"
+      end
+
+      log_message + "\n"
+    end
+
     private
       def formatted_dt
         @formatted_dt ||= time.iso8601(DT_PRECISION)

data/lib/timber/logger.rb

@@ -115,7 +115,8 @@ module Timber
 
       # This method is invoked when a log event occurs
       def call(severity, timestamp, progname, msg)
-        "#{String === msg ? msg : msg.inspect}\n"
+        log_entry = build_log_entry(severity, timestamp, progname, msg)
+        log_entry.to_s
       end
     end
 
@@ -197,7 +198,7 @@ module Timber
 
       self.level = environment_level
 
-      after_initialize if respond_to? :after_initialize
+      after_initialize if respond_to?(:after_initialize)
 
       @initialized = true
     end

data/lib/timber/version.rb

@@ -1,3 +1,3 @@
 module Timber
-  VERSION = "2.0.20"
+  VERSION = "2.0.21"
 end

data/spec/support/rails.rb

@@ -46,7 +46,12 @@ if defined?(::Rails)
         #
         # You can see here that they use simple class attribute, hence the reason we need
         # to update all of them: https://github.com/rails/rails/blob/700ec897f97c60016ad748236bf3a49ef15a20de/actionview/lib/action_view/base.rb#L157
-        Timber::Frameworks::Rails.set_logger(logger)
+        ::ActionCable::Server::Base.logger = logger if defined?(::ActionCable::Server::Base) && ::ActionCable::Server::Base.respond_to?(:logger=)
+        ::ActionController::Base.logger = logger if defined?(::ActionController::Base) && ::ActionController::Base.respond_to?(:logger=)
+        ::ActionMailer::Base.logger = logger if defined?(::ActionMailer::Base) && ::ActionMailer::Base.respond_to?(:logger=)
+        ::ActionView::Base.logger = logger if defined?(::ActionView::Base) && ::ActionView::Base.respond_to?(:logger=)
+        ::ActiveRecord::Base.logger = logger if defined?(::ActiveRecord::Base) && ::ActiveRecord::Base.respond_to?(:logger=)
+        ::Rails.logger = logger
 
         yield
 

data/spec/support/timber.rb

@@ -1,5 +1,4 @@
 require "timber"
 
 config = Timber::Config.instance
-config.append_metadata = true
-config.debug_logger = ::Logger.new(STDOUT)
+config.append_metadata = true

data/spec/timber/log_devices/http_spec.rb

@@ -11,7 +11,7 @@ describe Timber::LogDevices::HTTP do
       # Ensure that threads have not started
       thread = http.instance_variable_get(:@flush_thread)
       expect(thread).to be_nil
-      thread = http.instance_variable_get(:@outlet_thread)
+      thread = http.instance_variable_get(:@request_outlet_thread)
       expect(thread).to be_nil
     end
   end
@@ -23,6 +23,7 @@ describe Timber::LogDevices::HTTP do
     it "should buffer the messages" do
       http.write("test log message")
       expect(msg_queue.flush).to eq(["test log message"])
+      http.close
     end
 
     it "should start the flush threads" do
@@ -30,8 +31,10 @@ describe Timber::LogDevices::HTTP do
 
       thread = http.instance_variable_get(:@flush_thread)
       expect(thread).to be_alive
-      thread = http.instance_variable_get(:@outlet_thread)
+      thread = http.instance_variable_get(:@request_outlet_thread)
       expect(thread).to be_alive
+      expect(http).to receive(:flush).exactly(1).times
+      http.close
     end
 
     context "with a low batch size" do
@@ -41,6 +44,8 @@ describe Timber::LogDevices::HTTP do
         http.write("test")
         expect(http).to receive(:flush).exactly(1).times
         http.write("my log message")
+        expect(http).to receive(:flush).exactly(1).times
+        http.close
       end
     end
   end
@@ -54,7 +59,7 @@ describe Timber::LogDevices::HTTP do
       thread = http.instance_variable_get(:@flush_thread)
       sleep 0.1 # too fast!
       expect(thread).to_not be_alive
-      thread = http.instance_variable_get(:@outlet_thread)
+      thread = http.instance_variable_get(:@request_outlet_thread)
       sleep 0.1 # too fast!
       expect(thread).to_not be_alive
     end
@@ -86,13 +91,6 @@ describe Timber::LogDevices::HTTP do
       message_queue = http.instance_variable_get(:@msg_queue)
       expect(message_queue.size).to eq(0)
     end
-
-    it "should preserve formatting for mshpack payloads" do
-      http = described_class.new("MYKEY", flush_continuously: false)
-      http.write("This is a log message 1".to_msgpack)
-      http.write("This is a log message 2".to_msgpack)
-      http.send(:flush)
-    end
   end
 
   # Testing a private method because it helps break down our tests
@@ -100,18 +98,17 @@ describe Timber::LogDevices::HTTP do
     it "should start a intervaled flush thread and flush on an interval" do
       http = described_class.new("MYKEY", flush_interval: 0.1)
       http.send(:ensure_flush_threads_are_started)
-      expect(http).to receive(:flush).exactly(1).times
-      sleep 0.12 # too fast!
-      expect(http).to receive(:flush).exactly(1).times
-      sleep 0.12 # too fast!
+      expect(http).to receive(:flush).exactly(2).times
+      sleep 0.15 # too fast!
+      http.close
     end
   end
 
   # Outlet
-  describe "#outlet" do
+  describe "#request_outlet" do
     let(:time) { Time.utc(2016, 9, 1, 12, 0, 0) }
 
-    it "should start a intervaled flush thread and flush on an interval" do
+    it "should deliver requests on an interval" do
       stub = stub_request(:post, "https://logs.timber.io/frames").
         with(
           :body => start_with("\x92\x85\xA5level\xA4INFO\xA2dt\xBB2016-09-01T12:00:00.000000Z\xA7message\xB2test log message 1\xA7context\x81\xA6system".force_encoding("ASCII-8BIT")),
@@ -124,13 +121,37 @@ describe Timber::LogDevices::HTTP do
         to_return(:status => 200, :body => "", :headers => {})
 
       http = described_class.new("MYKEY", flush_interval: 0.1)
-      log_entry = Timber::LogEntry.new("INFO", time, nil, "test log message 1", nil, nil)
-      http.write(log_entry)
-      log_entry = Timber::LogEntry.new("INFO", time, nil, "test log message 2", nil, nil)
-      http.write(log_entry)
-      sleep 0.3
+      log_entry1 = Timber::LogEntry.new("INFO", time, nil, "test log message 1", nil, nil)
+      http.write(log_entry1)
+      log_entry2 = Timber::LogEntry.new("INFO", time, nil, "test log message 2", nil, nil)
+      http.write(log_entry2)
+      sleep 1
 
       expect(stub).to have_been_requested.times(1)
+
+      http.close
+    end
+  end
+
+  describe "#deliver_requests" do
+    it "should handle exceptions properly and return" do
+      allow_any_instance_of(Net::HTTP).to receive(:request).and_raise("boom")
+
+      http_device = described_class.new("MYKEY", flush_continuously: false)
+      req_queue = http_device.instance_variable_get(:@request_queue)
+
+      # Place a request on the queue
+      req = Net::HTTP::Post.new("/")
+      req_queue.enq(req)
+
+      # Start a HTTP connection to test the method directly
+      http = http_device.send(:build_http)
+      http.start do |conn|
+        result = http_device.send(:deliver_requests, conn)
+        expect(result).to eq(false)
+      end
+
+      expect(req_queue.size).to eq(1)
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: timber
 version: !ruby/object:Gem::Version
-  version: 2.0.20
+  version: 2.0.21
 platform: ruby
 authors:
 - Timber Technologies, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-19 00:00:00.000000000 Z
+date: 2017-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -199,7 +199,6 @@ files:
 - lib/timber/log_devices/http.rb
 - lib/timber/log_entry.rb
 - lib/timber/logger.rb
-- lib/timber/logger/metadata_methods.rb
 - lib/timber/overrides.rb
 - lib/timber/overrides/lograge.rb
 - lib/timber/overrides/rails_stdout_logging.rb
@@ -264,7 +263,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.4.5.2
 signing_key: 
 specification_version: 4
 summary: Log Better. Solve Problems Faster. https://timber.io

data/lib/timber/logger/metadata_methods.rb

@@ -1 +0,0 @@
-metadata_methods.rb