logtail-ruby 0.1.0

data/lib/logtail/log_devices/http/flushable_dropping_sized_queue.rb

@@ -0,0 +1,52 @@
+module Logtail
+  module LogDevices
+    class HTTP
+      # A simple thread-safe queue implementation that provides a #flush method.
+      # The built-in ruby `Queue` class does not provide a #flush method that allows
+      # the caller to retrieve all items on the queue in one call. The Ruby `SizedQueue` also
+      # implements thread waiting, which is something we want to avoid. To keep things
+      # simple and straight-forward, we designed this queue class.
+      # @private
+      class FlushableDroppingSizedQueue
+        def initialize(max_size)
+          @lock = Mutex.new
+          @max_size = max_size
+          @array = []
+        end
+
+        # Adds a message to the queue
+        def enq(msg)
+          @lock.synchronize do
+            if !full?
+              @array << msg
+            end
+          end
+        end
+
+        # Removes a single item from the queue
+        def deq
+          @lock.synchronize do
+            @array.pop
+          end
+        end
+
+        # Flushes all message from the queue and returns them.
+        def flush
+          @lock.synchronize do
+            old = @array
+            @array = []
+            return old
+          end
+        end
+
+        def full?
+          size >= @max_size
+        end
+
+        def size
+          @array.size
+        end
+      end
+    end
+  end
+end

data/lib/logtail/log_devices/http/request_attempt.rb

@@ -0,0 +1,20 @@
+module Logtail
+  module LogDevices
+    class HTTP
+      # Represents an attempt to deliver a request. Requests can be retried, hence
+      # why we keep track of the number of attempts.
+      class RequestAttempt
+        attr_reader :attempts, :request
+
+        def initialize(req)
+          @attempts = 0
+          @request = req
+        end
+
+        def attempted!
+          @attempts += 1
+        end
+      end
+    end
+  end
+end

data/lib/logtail/log_entry.rb

@@ -0,0 +1,110 @@
+require "socket"
+require "time"
+
+require "logtail/contexts"
+require "logtail/events"
+
+module Logtail
+  # Represents a new log entry into the log. This is an intermediary class between
+  # `Logger` and the log device that you set it up with.
+  class LogEntry #:nodoc:
+    BINARY_LIMIT_THRESHOLD = 1_000.freeze
+    DT_PRECISION = 6.freeze
+    MESSAGE_MAX_BYTES = 8192.freeze
+
+    attr_reader :context_snapshot, :event, :level, :message, :progname, :tags, :time
+
+    # Creates a log entry suitable to be sent to the Logtail API.
+    # @param level [Integer] the log level / severity
+    # @param time [Time] the exact time the log message was written
+    # @param progname [String] the progname scope for the log message
+    # @param message [String] Human readable log message.
+    # @param context_snapshot [Hash] structured data representing a snapshot of the context at
+    #   the given point in time.
+    # @param event [Logtail.Event] structured data representing the log line event. This should be
+    #   an instance of {Logtail.Event}.
+    # @return [LogEntry] the resulting LogEntry object
+    def initialize(level, time, progname, message, context_snapshot, event, options = {})
+      @level = level
+      @time = time.utc
+      @progname = progname
+
+      # If the message is not a string we call inspect to ensure it is a string.
+      # This follows the default behavior set by ::Logger
+      # See: https://github.com/ruby/ruby/blob/trunk/lib/logger.rb#L615
+      @message = message.is_a?(String) ? message : message.inspect
+      @message = @message.byteslice(0, MESSAGE_MAX_BYTES)
+      @tags = options[:tags]
+      @context_snapshot = context_snapshot
+      @event = event
+    end
+
+    # Builds a hash representation containing simple objects, suitable for serialization (JSON).
+    def to_hash(options = {})
+      options ||= {}
+      hash = {
+        :level => level,
+        :dt => formatted_dt,
+        :message => message
+      }
+
+      if !tags.nil? && tags.length > 0
+        hash[:tags] = tags
+      end
+
+      if !event.nil?
+        hash.merge!(event)
+      end
+
+      if !context_snapshot.nil? && context_snapshot.length > 0
+        hash[:context] = context_snapshot
+      end
+
+      if options[:only]
+        hash.select do |key, _value|
+          options[:only].include?(key)
+        end
+      elsif options[:except]
+        hash.select do |key, _value|
+          !options[:except].include?(key)
+        end
+      else
+        hash
+      end
+    end
+
+    def inspect
+      to_s
+    end
+
+    def to_json(options = {})
+      to_hash.to_json
+    end
+
+    def to_msgpack(*args)
+      to_hash.to_msgpack(*args)
+    end
+
+    # This is used when LogEntry objects make it to a non-Logtail logger.
+    def to_s
+      message + "\n"
+    end
+
+    private
+      def formatted_dt
+        @formatted_dt ||= time.iso8601(DT_PRECISION)
+      end
+
+      # Attempts to encode a non UTF-8 string into UTF-8, discarding invalid characters.
+      # If it fails, a nil is returned.
+      def encode_string(string)
+        string.encode('UTF-8', {
+          :invalid => :replace,
+          :undef   => :replace,
+          :replace => '?'
+        })
+      rescue Exception
+        nil
+      end
+  end
+end

data/lib/logtail/logger.rb

@@ -0,0 +1,270 @@
+require "logger"
+require "msgpack"
+
+require "logtail/config"
+require "logtail/current_context"
+require "logtail/log_devices"
+require "logtail/log_entry"
+
+module Logtail
+  # The Logtail Logger behaves exactly like the standard Ruby `::Logger`, except that it supports a
+  # transparent API for logging structured data and events.
+  #
+  # @example Basic logging
+  #   logger.info "Payment rejected for customer #{customer_id}"
+  #
+  # @example Logging an event
+  #   logger.info "Payment rejected", payment_rejected: {customer_id: customer_id, amount: 100}
+  class Logger < ::Logger
+
+    # @private
+    class Formatter
+      # Formatters get the formatted level from the logger.
+      SEVERITY_MAP = {
+        "DEBUG" => :debug,
+        "INFO" => :info,
+        "WARN" => :warn,
+        "ERROR" => :error,
+        "FATAL" => :fatal,
+        "UNKNOWN" => :unknown
+      }
+      EMPTY_ARRAY = []
+
+      private
+        def build_log_entry(severity, time, progname, logged_obj)
+          context_snapshot = CurrentContext.instance.snapshot
+          level = SEVERITY_MAP.fetch(severity)
+          tags = extract_active_support_tagged_logging_tags
+
+          if logged_obj.is_a?(Event)
+            LogEntry.new(level, time, progname, logged_obj.message, context_snapshot, logged_obj,
+                         tags: tags)
+          elsif logged_obj.is_a?(Hash)
+            # Extract the tags
+            tags = tags.clone
+            tags.push(logged_obj.delete(:tag)) if logged_obj.key?(:tag)
+            tags.concat(logged_obj.delete(:tags)) if logged_obj.key?(:tags)
+            tags.uniq!
+
+            message = logged_obj.delete(:message)
+
+            LogEntry.new(level, time, progname, message, context_snapshot, logged_obj, tags: tags)
+          else
+            LogEntry.new(level, time, progname, logged_obj, context_snapshot, nil, tags: tags)
+          end
+        end
+
+        # Because of all the crazy ways Rails has attempted tags, we need this crazy method.
+        def extract_active_support_tagged_logging_tags
+          Thread.current[:activesupport_tagged_logging_tags] ||
+            Thread.current[tagged_logging_object_key_name] ||
+            EMPTY_ARRAY
+        end
+
+        def tagged_logging_object_key_name
+          @tagged_logging_object_key_name ||= "activesupport_tagged_logging_tags:#{object_id}"
+        end
+    end
+
+    # For use in development and test environments where you do not want metadata
+    # included in the log lines.
+    class MessageOnlyFormatter < Formatter
+      # This method is invoked when a log event occurs
+      def call(severity, timestamp, progname, msg)
+        log_entry = build_log_entry(severity, timestamp, progname, msg)
+        log_entry.to_s
+      end
+    end
+
+    # Structures your log messages as strings and appends metadata if
+    # `Logtail::Config.instance.append_metadata?` is true.
+    #
+    # Example message with metdata:
+    #
+    #   My log message @metadata {"level":"info","dt":"2016-09-01T07:00:00.000000-05:00"}
+    #
+    class AugmentedFormatter < Formatter
+      METADATA_CALLOUT = " @metadata ".freeze
+      NEW_LINE = "\n".freeze
+      ESCAPED_NEW_LINE = "\\n".freeze
+
+      def call(severity, time, progname, msg)
+        log_entry = build_log_entry(severity, time, progname, msg)
+        metadata = log_entry.to_json(:except => [:message])
+        # use << for concatenation for performance reasons
+        log_entry.message.gsub(NEW_LINE, ESCAPED_NEW_LINE) << METADATA_CALLOUT <<
+          metadata << NEW_LINE
+      end
+    end
+
+    # Structures your log messages into JSON.
+    #
+    #   logger = Logtail::Logger.new(STDOUT)
+    #   logger.formatter = Logtail::JSONFormatter.new
+    #
+    # Example message:
+    #
+    #   {"level":"info","dt":"2016-09-01T07:00:00.000000-05:00","message":"My log message"}
+    #
+    class JSONFormatter < Formatter
+      def call(severity, time, progname, msg)
+        # use << for concatenation for performance reasons
+        build_log_entry(severity, time, progname, msg).to_json << "\n"
+      end
+    end
+
+    # Passes through the LogEntry object. This is specifically used for the {Logtail::LogDevices::HTTP}
+    # class. This allows the IO device to format it however it wants. This is necessary for
+    # MessagePack because it requires a fixed array size before encoding. And since HTTP is
+    # sending data in batches, the encoding should happen there.
+    class PassThroughFormatter < Formatter
+      def call(severity, time, progname, msg)
+        build_log_entry(severity, time, progname, msg)
+      end
+    end
+
+    # Creates a new Logtail::Logger instance where the passed argument is an IO device. That is,
+    # anything that responds to `#write` and `#close`.
+    #
+    # Note, this method does *not* accept the same arguments as the standard Ruby `::Logger`.
+    # The Ruby `::Logger` accepts additional options controlling file rotation if the first argument
+    # is a file *name*. This is a design flaw that Logtail does not assume. Logging to a file, or
+    # multiple IO devices is demonstrated in the examples below.
+    #
+    # @example Logging to STDOUT
+    #   logger = Logtail::Logger.new(STDOUT)
+    #
+    # @example Logging to the Logtail HTTP device
+    #   http_device = Logtail::LogDevices::HTTP.new("my-logtail-api-key")
+    #   logger = Logtail::Logger.new(http_device)
+    #
+    # @example Logging to a file (with rotation)
+    #   file_device = Logger::LogDevice.new("path/to/file.log")
+    #   logger = Logtail::Logger.new(file_device)
+    #
+    # @example Logging to a file and the Logtail HTTP device (multiple log devices)
+    #   http_device = Logtail::LogDevices::HTTP.new("my-logtail-api-key")
+    #   file_logger = ::Logger.new("path/to/file.log")
+    #   logger = Logtail::Logger.new(http_device, file_logger)
+    def initialize(*io_devices_and_loggers)
+      if io_devices_and_loggers.size == 0
+        raise ArgumentError.new("At least one IO device or Logger must be provided when " +
+          "instantiating a Logtail::Logger. Ex: Logtail::Logger.new(STDOUT).")
+      end
+
+      @extra_loggers = io_devices_and_loggers[1..-1].collect do |obj|
+        if is_a_logger?(obj)
+          obj
+        else
+          self.class.new(obj)
+        end
+      end
+
+      io_device = io_devices_and_loggers[0]
+
+      super(io_device)
+
+      # Ensure we sync STDOUT to avoid buffering
+      if io_device.respond_to?(:"sync=")
+        io_device.sync = true
+      end
+
+      # Set the default formatter. The formatter cannot be set during
+      # initialization, and can be changed with #formatter=.
+      if io_device.is_a?(LogDevices::HTTP)
+        self.formatter = PassThroughFormatter.new
+      elsif Config.instance.development? || Config.instance.test?
+        self.formatter = MessageOnlyFormatter.new
+      else
+        self.formatter = JSONFormatter.new
+      end
+
+      self.level = environment_level
+
+      after_initialize if respond_to?(:after_initialize)
+
+      Logtail::Config.instance.debug { "Logtail::Logger instantiated, level: #{level}, formatter: #{formatter.class}" }
+
+      @initialized = true
+    end
+
+    # Sets a new formatted on the logger.
+    #
+    # @note The formatter cannot be changed if you are using the HTTP logger backend.
+    def formatter=(value)
+      if @initialized && @logdev && @logdev.dev.is_a?(Logtail::LogDevices::HTTP) && !value.is_a?(PassThroughFormatter)
+        raise ArgumentError.new("The formatter cannot be changed when using the " +
+          "Logtail::LogDevices::HTTP log device. The PassThroughFormatter must be used for proper " +
+          "delivery.")
+      end
+
+      super
+    end
+
+    def level=(value)
+      if value.is_a?(Symbol)
+        value = level_from_symbol(value)
+      end
+      super
+    end
+
+    # @private
+    def with_context(context, &block)
+      Logtail::CurrentContext.with(context, &block)
+    end
+
+    # Patch to ensure that the {#level} method is used instead of `@level`.
+    # This is required because of Rails' monkey patching on Logger via `::LoggerSilence`.
+    def add(severity, message = nil, progname = nil, &block)
+      return true if @logdev.nil? || (severity || UNKNOWN) < level
+
+      @extra_loggers.each do |logger|
+        logger.add(severity, message, progname, &block)
+      end
+
+      super
+    end
+
+    # Backwards compatibility with older ActiveSupport::Logger versions
+    Logger::Severity.constants.each do |severity|
+      class_eval(<<-EOT, __FILE__, __LINE__ + 1)
+        def #{severity.downcase}(*args, &block)
+          progname = args.first
+          options = args.last
+
+          if args.length == 2 and options.is_a?(Hash) && options.length > 0
+            progname = options.merge(message: progname)
+          end
+
+          add(#{severity}, nil, progname, &block)
+        end
+
+        def #{severity.downcase}?                # def debug?
+          Logger::#{severity} >= level           #   DEBUG >= level
+        end                                      # end
+      EOT
+    end
+
+    private
+      def environment_level
+        level = ([ENV['LOG_LEVEL'].to_s.upcase, "DEBUG"] & %w[DEBUG INFO WARN ERROR FATAL UNKNOWN]).compact.first
+        self.class.const_get(level)
+      end
+
+      def level_from_symbol(value)
+        case value
+        when :debug; DEBUG
+        when :info;  INFO
+        when :warn;  WARN
+        when :error; ERROR
+        when :fatal; FATAL
+        when :unknown; UNKNOWN
+        else; raise ArgumentError.new("level #{value.inspect} is not a valid logger level")
+        end
+      end
+
+      def is_a_logger?(obj)
+        obj.respond_to?(:debug) && obj.respond_to?(:info) && obj.respond_to?(:warn)
+      end
+  end
+end