timberio 1.0.0.beta1

data/lib/timber/log_devices/heroku_logplex.rb

@@ -0,0 +1,14 @@
+require File.join(File.dirname(__FILE__), "heroku_logplex", "hybrid_formatter")
+
+module Timber
+  module LogDevices
+    class HerokuLogplex < IO
+      def initialize(_options = {})
+        super(STDOUT)
+        if formatter.is_a?(IO::HybridFormatter)
+          formatter.extend HybridFormatter
+        end
+      end
+    end
+  end
+end

data/lib/timber/log_devices/http/log_pile.rb

@@ -0,0 +1,86 @@
+require "thread"
+
+module Timber
+  module LogDevices
+    class HTTP < LogDevice
+      # This is a thread safe queue for transporting logs to the Timber API.
+      # TODO: Have these log lines persist to a file where
+      #       a daemon can pick them up.
+      class LogPile
+        class << self
+          def each(&block)
+            instances.values.each(&block)
+          end
+
+          def get(application_key)
+            instances[application_key] ||= new(application_key)
+          end
+
+          private
+            def instances
+              @instances ||= {}
+            end
+        end
+
+        attr_reader :application_key
+
+        def initialize(application_key)
+          @application_key = application_key
+          @mutex = Mutex.new
+        end
+
+        def drop(log_line)
+          mutex.synchronize do
+            log_lines << log_line
+          end
+        rescue LogLine::InvalidMessageError => e
+          # Ignore the error and log it.
+          Config.logger.error(e)
+        rescue Exception => e
+          # Fail safe to ensure the Timber gem never fails the app.
+          Config.logger.exception(e)
+        end
+
+        def empty(&_block)
+          if log_lines.any?
+            copy = log_lines_copy
+            yield(copy) if block_given?
+            remove(copy)
+            self
+          end
+        end
+
+        def size
+          log_lines.size
+        end
+
+        private
+          def mutex
+            @mutex
+          end
+
+          def remove(log_lines_copy)
+            mutex.synchronize do
+              # Delete items by object_id since we are working
+              # with the same object. Do not use equality here.
+              log_lines_copy.each do |l1|
+                log_lines.delete_if { |l2| l2.object_id == l1.object_id }
+              end
+            end
+          end
+
+          def log_lines_copy
+            mutex.synchronize do
+              # Copy the array structure so we aren't dealing with
+              # a changing array, but do not copy the items.
+              log_lines.clone
+            end
+          end
+
+          def log_lines
+            @log_lines ||= []
+          end
+      end
+    end
+  end
+end

data/lib/timber/log_devices/http/log_truck/delivery.rb

@@ -0,0 +1,116 @@
+require "base64"
+require "net/http"
+require "net/https"
+require "uri"
+
+module Timber
+  module LogDevices
+    class HTTP < LogDevice
+      class LogTruck
+        class Delivery
+          class DeliveryError < StandardError; end
+
+          API_URI = URI.parse("https://timber-odin.herokuapp.com/agent_log_frames")
+          CONTENT_TYPE = 'application/json'.freeze
+          READ_TIMEOUT_SECONDS = 35.freeze
+          RETRY_BACKOFF_SECONDS = 1.freeze
+          RETRY_COUNT = 4.freeze
+          USER_AGENT = "Timber Ruby Gem/#{Timber::VERSION}".freeze
+
+          HTTPS = Net::HTTP.new(API_URI.host, API_URI.port).tap do |https|
+            https.use_ssl = true
+            https.read_timeout = READ_TIMEOUT_SECONDS
+          end
+
+          attr_reader :application_key, :log_lines
+
+          def initialize(application_key, log_lines)
+            @application_key = application_key
+            @log_lines = log_lines
+          end
+
+          def deliver!(retry_count = 0)
+            Config.logger.debug("Attempting delivery of: #{body_json}")
+            request!
+          # Catch them all because of all the unknown exceptions that can happen during
+          # a http request.
+          rescue Exception => e
+            # Ensure that we are always returning a consistent error.
+            # This ensures we handle it appropriately and don't kill the
+            # thread above.
+            Config.logger.warn("Failed delivery: #{e.message}")
+
+            retry_count += 1
+            if retry_count <= RETRY_COUNT
+              backoff_seconds = RETRY_BACKOFF_SECONDS ** retry_count
+              Config.logger.warn("Backing off #{backoff_seconds} seconds")
+              sleep backoff_seconds
+              Config.logger.warn("Retrying, attempt #{retry_count}")
+              deliver!(retry_count)
+            else
+              Config.logger.warn("Retry attempts exceeded, dropping logs")
+              raise DeliveryError.new(e.message)
+            end
+          end
+
+          private
+            def https
+              @https ||= HTTPS
+            end
+
+            def request!
+              https.request(new_request).tap do |res|
+                code = res.code.to_i
+                if code < 200 || code >= 300
+                  raise DeliveryError.new("Bad response from Timber API - #{res.code}: #{res.body}")
+                end
+                Config.logger.debug("Success! #{code}: #{res.body}")
+              end
+            end
+
+            def new_request
+              Net::HTTP::Post.new(API_URI.request_uri).tap do |req|
+                req['Authorization'] = authorization_payload
+                req['Body-Checksum'] = body_checksum # the API checks for duplicate requests
+                req['Content-Type'] = CONTENT_TYPE
+                req['Log-Line-Count'] = log_lines.size # additional check to ensure the correct # of log lines were sent
+                req['User-Agent'] = USER_AGENT
+                req.body = body_json
+              end
+            end
+
+            # Used by the API to check for duplicate requests.
+            def body_checksum
+              @body_checksum ||= Digest::MD5.hexdigest(body_json)
+            end
+
+            def body_json
+              return @body_json if defined?(@body_json)
+              # Build the json as a string since it is more efficient.
+              # We are also working with string upstream for the same reason.
+              @body_json ||= <<-JSON
+                {"agent_log_frame": {"log_lines": #{log_lines_json}}}
+              JSON
+              @body_json.strip!
+              @body_json
+            end
+
+            def log_lines_json
+              return @log_lines_json if defined?(@log_lines_json)
+              @log_lines_json = "["
+              last_index = log_lines.size - 1
+              log_lines.each_with_index do |log_line, index|
+                @log_lines_json += log_line.to_json
+                @log_lines_json += ", " if index != last_index
+              end
+              @log_lines_json += "]"
+            end
+
+            def authorization_payload
+              @authorization_payload ||= "Basic #{Base64.strict_encode64(application_key).chomp}"
+            end
+        end
+      end
+    end
+  end
+end

data/lib/timber/log_devices/http/log_truck.rb

@@ -0,0 +1,87 @@
+require "uri"
+require "net/http"
+require "net/https"
+require File.join(File.dirname(__FILE__), "log_truck", "delivery")
+
+module Timber
+  module LogDevices
+    class HTTP < LogDevice
+      # Temporary class for alpha / beta purposes.
+      # Log lines will be written to a file where a daemon
+      # will pick them up. Most of this code will be moved
+      # to that daemon.
+      class LogTruck
+        THROTTLE_SECONDS = 3.freeze
+
+        class NoPayloadError < ArgumentError; end
+
+        class << self
+          def start!(options = {}, &_block)
+            return if @thread && @thread.alive?
+
+            # Old school options to support ruby 1.9 :(
+            options[:throttle_seconds] = THROTTLE_SECONDS if !options.key?(:throttle_seconds)
+            Config.logger.debug("Starting log truck with a #{options[:throttle_seconds]} second throttle")
+
+            # A new thread for looping and monitoring. We need to
+            # use a thread so that we can share memory.
+            @thread = Thread.new do
+              # ensure we always deliver upon exiting
+              at_exit { deliver }
+
+              # Keep looking for logs
+              loop do
+                deliver
+
+                # Yield a block, primarily for testing purposes
+                yield(Thread.current) if block_given?
+
+                # Throttle to reduce checking the pile
+                sleep options[:throttle_seconds]
+              end
+            end
+
+          rescue Exception => e
+            # failsafe to ensure we don't kill the app
+            Config.logger.exception(e)
+          end
+
+          # Deliver, return LogTruck object, otherwise
+          # raise an error.
+          def deliver
+            log_truck = nil
+            LogPile.each do |log_pile|
+              log_pile.empty do |log_lines|
+                # LogPile only empties if no exception is raised
+                begin
+                  # This will retry a number of times. If we can't get it during the retries
+                  # we drop the logs. Note, this strategy will improve when we write to a file
+                  # and use an actual agent.
+                  log_truck = new(log_pile.application_key, log_lines).tap(&:deliver!)
+                rescue Delivery::DeliveryError => e
+                  Config.logger.exception(e)
+                  # TODO: How do we handle server timeouts? The request could have still been processed.
+                end
+              end
+            end
+            log_truck
+          end
+        end
+
+        attr_reader :application_key, :log_lines
+
+        def initialize(application_key, log_lines)
+          if log_lines.empty?
+            raise NoPayloadError.new("a truck must contain a payload (at least one log line)")
+          end
+          @application_key = application_key
+          @log_lines = log_lines
+        end
+
+        def deliver!
+          Delivery.new(application_key, log_lines).deliver!
+        end
+      end
+    end
+  end
+end

data/lib/timber/log_devices/http.rb

@@ -0,0 +1,28 @@
+require File.join(File.dirname(__FILE__), "http", "log_pile")
+require File.join(File.dirname(__FILE__), "http", "log_truck")
+
+module Timber
+  module LogDevices
+    class HTTP < LogDevice
+      SPLIT_LINES = false
+
+      attr_reader :application_key
+
+      def initialize(application_key = nil)
+        @application_key = application_key || Config.application_key
+        if @application_key.nil?
+          raise ArgumentError.new("A Timber application_key is required")
+        end
+        LogTruck.start!
+      end
+
+      def close(*args)
+      end
+
+      private
+        def write_log_line(log_line)
+          LogPile.get(application_key).drop(log_line)
+        end
+    end
+  end
+end

data/lib/timber/log_devices/io/formatter.rb

@@ -0,0 +1,46 @@
+module Timber
+  module LogDevices
+    class IO < LogDevice
+      class Formatter
+        # Do not change this, the API matches on it. Otherwise nothing
+        # get parsed.
+        CALLOUT     = "@timber.io "
+        CALLOUT_END = "@original "
+
+        # Embed in a String to clear all previous ANSI sequences.
+        CLEAR   = "\e[0m"
+        BOLD    = "\e[1m"
+
+        # Colors
+        BLACK     = "\e[30m"
+        DARK_GRAY = "\e[1;30m"
+        RED       = "\e[31m"
+        GREEN     = "\e[32m"
+        YELLOW    = "\e[33m"
+        BLUE      = "\e[34m"
+        MAGENTA   = "\e[35m"
+        CYAN      = "\e[36m"
+        WHITE     = "\e[37m"
+
+        def initialize(options = {})
+          @ansi_format = options.key?(:ansi_format) ? options[:ansi_format] == true : true
+        end
+
+        def ansi_format?
+          @ansi_format == true
+        end
+
+        def format(_log_line)
+          raise NotImplementedError.new("#format is not implemented")
+        end
+
+        private
+          def ansi_format(*args)
+            text = args.pop
+            return text unless ansi_format?
+            "#{args.join}#{text}#{CLEAR}"
+          end
+      end
+    end
+  end
+end

data/lib/timber/log_devices/io/hybrid_formatter.rb

@@ -0,0 +1,41 @@
+module Timber
+  module LogDevices
+    class IO < LogDevice
+      class HybridFormatter < Formatter
+        def initialize(options = {})
+          super
+          @date_prefix = options.key?(:date_prefix) ? options[:date_prefix] : false
+        end
+
+        def date_prefix?
+          @date_prefix == true
+        end
+
+        def format(log_line)
+          "#{log_line.message}#{context_message(log_line)}"
+        end
+
+        private
+          def base_message(log_line)
+            text = ""
+            if date_prefix?
+              text << "#{log_line.formatted_dt} "
+            end
+            text << log_line.message
+            text
+          end
+
+          def context_message(log_line)
+            # The callout must be before the formatting, otherwise we leave
+            # the message ending with a color formatting and not a reset.
+            # Anything before the callout modifies the original message.
+            CALLOUT + ansi_format(DARK_GRAY, encoded_context(log_line))
+          end
+
+          def encoded_context(log_line)
+            log_line.context_snapshot.to_logfmt
+          end
+      end
+    end
+  end
+end

data/lib/timber/log_devices/io/hybrid_hidden_formatter.rb

@@ -0,0 +1,36 @@
+module Timber
+  module LogDevices
+    class IO < LogDevice
+      class HybridHiddenFormatter < HybridFormatter
+        CLEAR_SEQUENCE = "\e8\e[K".freeze
+        CLEAR_STEP_SIZE = 20.freeze
+        SAVE_CURSOR_POSITION = "\e7".freeze
+
+        def format(log_line)
+          "#{SAVE_CURSOR_POSITION}#{context_message(log_line)}#{base_message(log_line)}"
+        end
+
+        private
+          def context_message(log_line)
+            text = encoded_context(log_line)
+            position = CLEAR_STEP_SIZE
+            sequence_size = CLEAR_SEQUENCE.size
+            step_size = sequence_size + CLEAR_STEP_SIZE
+            while position < text.length
+              # ensure we don't insert before a \
+              while text[position - 1] == "\\"
+                position += 1
+              end
+              text.insert(position, CLEAR_SEQUENCE)
+              position += step_size
+            end
+            ansi_format(DARK_GRAY, "#{CALLOUT}#{CLEAR_SEQUENCE}#{text} #{CALLOUT_END}#{CLEAR_SEQUENCE}")
+          end
+
+          def encoded_context(log_line)
+            log_line.context_snapshot.to_logfmt
+          end
+      end
+    end
+  end
+end

data/lib/timber/log_devices/io/json_formatter.rb

@@ -0,0 +1,11 @@
+module Timber
+  module LogDevices
+    class IO < LogDevice
+      class JSONFormatter < Formatter
+        def format(log_line)
+          CALLOUT + log_line.to_json
+        end
+      end
+    end
+  end
+end

data/lib/timber/log_devices/io/logfmt_formatter.rb

@@ -0,0 +1,11 @@
+module Timber
+  module LogDevices
+    class IO < LogDevice
+      class LogfmtFormatter < Formatter
+        def format(log_line)
+          @CALLOUT + log_line.to_logfmt
+        end
+      end
+    end
+  end
+end

data/lib/timber/log_devices/io.rb

@@ -0,0 +1,41 @@
+require File.join(File.dirname(__FILE__), "io", "formatter")
+require File.join(File.dirname(__FILE__), "io", "hybrid_formatter")
+require File.join(File.dirname(__FILE__), "io", "hybrid_hidden_formatter")
+require File.join(File.dirname(__FILE__), "io", "json_formatter")
+require File.join(File.dirname(__FILE__), "io", "logfmt_formatter")
+
+module Timber
+  module LogDevices
+    # The purpose of a Timber log device is to take the raw log message and enrich it
+    # with the current context.
+    #
+    # The IO log device works with any IO object. That is, any object that
+    # response to #write(message).
+    class IO < LogDevice
+      attr_reader :formatter
+
+      # Instantiates a new Timber IO log device.
+      #
+      # @param io [IO] any object the responds to #write(message)
+      def initialize(io = STDOUT, options = {})
+        io.sync = true if io.respond_to?(:sync=) # ensures logs are written immediately instead of being buffered by ruby
+        @formatter = options[:formatter] || HybridHiddenFormatter.new
+        @io = io
+      end
+
+      def close(*_args)
+        io.close
+      end
+
+      private
+        def write_log_line(log_line)
+          formatted_message = formatter.format(log_line)
+          io.write(formatted_message + "\n")
+        end
+
+        def io
+          @io
+        end
+    end
+  end
+end

data/lib/timber/log_devices.rb

@@ -0,0 +1,4 @@
+# Order is relevant
+require File.join(File.dirname(__FILE__), "log_devices", "io")
+require File.join(File.dirname(__FILE__), "log_devices", "heroku_logplex")
+require File.join(File.dirname(__FILE__), "log_devices", "http")

data/lib/timber/log_line.rb

@@ -0,0 +1,33 @@
+module Timber
+  class LogLine
+    include Patterns::ToJSON
+    include Patterns::ToLogfmt
+
+    # Raised when there is an issue with the message being passed.
+    # Note: this is handled in Logger
+    class InvalidMessageError < ArgumentError; end
+
+    attr_reader :context_snapshot, :dt, :line_indexes, :message
+
+    def initialize(message)
+      @dt = Time.now.utc # Capture the time as soon as possible
+      message = message.to_s
+      if message.bytesize > APISettings::MESSAGE_BYTE_SIZE_MAX
+        Config.logger.warn("Log line message is too long, truncating")
+        message = message.byteslice(0, APISettings::MESSAGE_BYTE_SIZE_MAX)
+      end
+      @message = message
+      CurrentLineIndexes.log_line_added(self) # Bump the indexes
+      @context_snapshot = CurrentContext.snapshot
+    end
+
+    def formatted_dt
+      @formatted_dt ||= Macros::DateFormatter.format(dt)
+    end
+
+    private
+      def json_payload
+        @json_payload ||= {:dt => formatted_dt, :message => message}.merge(context_snapshot.as_json)
+      end
+  end
+end

data/lib/timber/logger.rb

@@ -0,0 +1,20 @@
+require "logger"
+
+module Timber
+  # A simple interface to instantiate a logger. It does a couple of things:
+  # 1. Simplifies Rails logger instantiation across Rails versions. This
+  #    helps with simplifying the Readme / install instructions.
+  # 2. Serves as a placeholder should we want to extend the logger and add
+  #    Timber specific functionality.
+  module Logger
+    def self.new(logger_or_logdev = nil)
+      logger = if logger_or_logdev.is_a?(::Logger)
+        logger_or_logdev
+      else
+        Frameworks.logger(logger_or_logdev)
+      end
+      logger.extend(self)
+      logger
+    end
+  end
+end

data/lib/timber/macros/compactor.rb

@@ -0,0 +1,16 @@
+module Timber
+  module Macros
+    module Compactor
+      def self.compact(hash)
+        new_hash = {}
+        hash.each do |k, v|
+          deep_v = v.is_a?(Hash) ? compact(v) : v
+          if !deep_v.nil? && deep_v != [] && deep_v != {}
+            new_hash[k] = deep_v
+          end
+        end
+        new_hash
+      end
+    end
+  end
+end

data/lib/timber/macros/date_formatter.rb

@@ -0,0 +1,9 @@
+module Timber
+  module Macros
+    module DateFormatter
+      def self.format(dt)
+        dt.send(APISettings::DATE_FORMAT, APISettings::DATE_FORMAT_PRECISION)
+      end
+    end
+  end
+end

data/lib/timber/macros/deep_merger.rb

@@ -0,0 +1,11 @@
+module Timber
+  module Macros
+    # Deep merges hash keys
+    module DeepMerger
+      def self.merge(first, second)
+        merger = proc { |_key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
+        first.merge(second, &merger)
+      end
+    end
+  end
+end