logstash-logger-p 0.26.1

data/lib/logstash-logger/formatter/base.rb

@@ -0,0 +1,73 @@
+require 'logger'
+require 'socket'
+require 'time'
+
+module LogStashLogger
+  module Formatter
+    HOST = ::Socket.gethostname
+
+    class Base < ::Logger::Formatter
+      include ::LogStashLogger::TaggedLogging::Formatter
+
+      def initialize(customize_event: nil)
+        @customize_event = customize_event
+        super()
+      end
+
+      def call(severity, time, _progname, message)
+        event = build_event(message, severity, time)
+        format_event(event) unless event.cancelled?
+      end
+
+      private
+
+      def build_event(message, severity, time)
+        data = message
+        if data.is_a?(String) && data.start_with?('{'.freeze)
+          data = (JSON.parse(message) rescue nil) || message
+        end
+
+        event = case data
+                  when LogStash::Event
+                    data.clone
+                  when Hash
+                    event_data = data.clone
+                    event_data['message'.freeze] = event_data.delete(:message) if event_data.key?(:message)
+                    event_data['tags'.freeze] = event_data.delete(:tags) if event_data.key?(:tags)
+                    event_data['source'.freeze] = event_data.delete(:source) if event_data.key?(:source)
+                    event_data['type'.freeze] = event_data.delete(:type) if event_data.key?(:type)
+                    event_data['@timestamp'.freeze] = time
+                    LogStash::Event.new(event_data)
+                  else
+                    LogStash::Event.new("message".freeze => msg2str(data), "@timestamp".freeze => time)
+                end
+
+        event['severity'.freeze] ||= severity
+        #event.type = progname
+
+        event['host'.freeze] ||= HOST
+
+        current_tags.each { |tag| event.tag(tag) }
+
+        LogStashLogger.configuration.customize_event_block.call(event) if LogStashLogger.configuration.customize_event_block.respond_to?(:call)
+
+        @customize_event.call(event) if @customize_event
+
+        # In case Time#to_json has been overridden
+        if event.timestamp.is_a?(Time)
+          event.timestamp = event.timestamp.iso8601(3)
+        end
+
+        if LogStashLogger.configuration.max_message_size && event['message']
+          event['message'.freeze] = event['message'.freeze].byteslice(0, LogStashLogger.configuration.max_message_size)
+        end
+
+        event
+      end
+
+      def format_event(event)
+        event
+      end
+    end
+  end
+end

data/lib/logstash-logger/formatter/cee.rb

@@ -0,0 +1,11 @@
+module LogStashLogger
+  module Formatter
+    class Cee < Base
+      private
+
+      def format_event(event)
+        "@cee:#{event.to_json}"
+      end
+    end
+  end
+end

data/lib/logstash-logger/formatter/cee_syslog.rb

@@ -0,0 +1,22 @@
+module LogStashLogger
+  module Formatter
+    class CeeSyslog < Cee
+      def call(severity, time, progname, message)
+        @progname = progname
+        super
+      end
+
+      private
+
+      def build_facility(host)
+        facility = host.dup
+        facility << " #{@progname}" if @progname
+        facility
+      end
+
+      def format_event(event)
+        "#{build_facility(event["host".freeze])}:#{super}\n"
+      end
+    end
+  end
+end

data/lib/logstash-logger/formatter/json.rb

@@ -0,0 +1,11 @@
+module LogStashLogger
+  module Formatter
+    class Json < Base
+      private
+
+      def format_event(event)
+        event.to_json
+      end
+    end
+  end
+end

data/lib/logstash-logger/formatter/json_lines.rb

@@ -0,0 +1,11 @@
+module LogStashLogger
+  module Formatter
+    class JsonLines < Base
+      private
+
+      def format_event(event)
+        "#{event.to_json}\n"
+      end
+    end
+  end
+end

data/lib/logstash-logger/formatter/logstash_event.rb

@@ -0,0 +1,6 @@
+module LogStashLogger
+  module Formatter
+    class LogStashEvent < Base
+    end
+  end
+end

data/lib/logstash-logger/formatter.rb

@@ -0,0 +1,51 @@
+require 'logstash-logger/formatter/base'
+
+module LogStashLogger
+	module Formatter
+		DEFAULT_FORMATTER = :json_lines
+
+    autoload :LogStashEvent, 'logstash-logger/formatter/logstash_event'
+    autoload :Json, 'logstash-logger/formatter/json'
+    autoload :JsonLines, 'logstash-logger/formatter/json_lines'
+    autoload :Cee, 'logstash-logger/formatter/cee'
+    autoload :CeeSyslog, 'logstash-logger/formatter/cee_syslog'
+
+    def self.new(formatter_type, customize_event: nil)
+      build_formatter(formatter_type, customize_event)
+    end
+
+    def self.build_formatter(formatter_type, customize_event)
+      formatter_type ||= DEFAULT_FORMATTER
+
+      formatter = if custom_formatter_instance?(formatter_type)
+        formatter_type
+      elsif custom_formatter_class?(formatter_type)
+        formatter_type.new
+      else
+        formatter_klass(formatter_type).new(customize_event: customize_event)
+      end
+
+      formatter.send(:extend, ::LogStashLogger::TaggedLogging::Formatter)
+      formatter
+    end
+
+    def self.formatter_klass(formatter_type)
+      case formatter_type.to_sym
+      when :json_lines then JsonLines
+      when :json then Json
+      when :logstash_event then LogStashEvent
+      when :cee then Cee
+      when :cee_syslog then CeeSyslog
+      else fail ArgumentError, 'Invalid formatter'
+      end
+    end
+
+    def self.custom_formatter_instance?(formatter_type)
+      formatter_type.respond_to?(:call)
+    end
+
+    def self.custom_formatter_class?(formatter_type)
+      formatter_type.is_a?(Class) && formatter_type.method_defined?(:call)
+    end
+	end
+end

data/lib/logstash-logger/logger.rb

@@ -0,0 +1,106 @@
+require 'logger'
+require 'logstash-logger/tagged_logging'
+require 'logstash-logger/silenced_logging'
+
+module LogStashLogger
+  autoload :MultiLogger, 'logstash-logger/multi_logger'
+
+  def self.new(*args)
+    opts = extract_opts(*args)
+    build_logger(opts)
+  end
+
+  def self.extended(base)
+    base.instance_eval do
+      class << self
+        attr_reader :device
+      end
+
+      def flush
+        !!(@device.flush if @device.respond_to?(:flush))
+      end
+
+      def reset
+        @device.reset if @device.respond_to?(:reset)
+      end
+    end
+  end
+
+  protected
+
+  def self.extract_opts(*args)
+    args.flatten!
+
+    if args.length > 1
+      if args.all?{|arg| arg.is_a?(Hash)}
+        # Deprecated array of hashes
+        warn "[LogStashLogger] Passing an array of hashes to the constructor is deprecated. Please replace with an options hash: { type: :multi_delegator, outputs: [...] }"
+        { type: :multi_delegator, outputs: args }
+      else
+        # Deprecated host/port/type constructor
+        warn "[LogStashLogger] The (host, port, type) constructor is deprecated. Please use an options hash instead."
+        host, port, type = *args
+        { host: host, port: port, type: type }
+      end
+    elsif Hash === args[0]
+      args[0]
+    else
+      fail ArgumentError, "Invalid LogStashLogger options"
+    end
+  end
+
+  def self.build_logger(opts)
+    formatter = Formatter.new(opts.delete(:formatter), customize_event: opts.delete(:customize_event))
+
+    logger_type = opts[:type].to_s.to_sym
+    logger = case logger_type
+    when :multi_logger
+      build_multi_logger(opts)
+    when :syslog
+      build_syslog_logger(opts)
+    else
+      build_default_logger(opts)
+    end
+
+    logger.formatter = formatter if formatter
+    logger
+  end
+
+  private
+
+  def self.build_default_logger(opts)
+    logger_class = opts.delete(:logger_class) || ::Logger
+    device = Device.new(opts)
+    logger_class.new(device).tap do |logger|
+      logger.instance_variable_set(:@device, device)
+      extend_logger(logger)
+    end
+  end
+
+  def self.build_multi_logger(opts)
+    output_configurations = opts.delete(:outputs) || []
+    loggers = output_configurations.map do |config|
+      logger_opts = opts.merge(config)
+      build_logger(logger_opts)
+    end
+    MultiLogger.new(loggers)
+  end
+
+  def self.build_syslog_logger(opts)
+    logger = begin
+      require 'syslog/logger'
+
+      Syslog::Logger.new(opts[:program_name], opts[:facility])
+    rescue ArgumentError
+      Syslog::Logger.new(opts[:program_name])
+    end
+
+    extend_logger(logger)
+  end
+
+  def self.extend_logger(logger)
+    logger.extend(self)
+    logger.extend(TaggedLogging)
+    logger.extend(SilencedLogging)
+  end
+end

data/lib/logstash-logger/multi_logger.rb

@@ -0,0 +1,153 @@
+# Adapted from https://github.com/ffmike/multilogger
+module LogStashLogger
+  class MultiLogger < ::Logger
+    # Array of Loggers to be logged to. These can be anything that acts reasonably like a Logger.
+    attr_accessor :loggers
+
+    def level=(value)
+      super
+      @loggers.each do |logger|
+        logger.level = value
+      end
+    end
+
+    def progname=(value)
+      super
+      @loggers.each do |logger|
+        logger.progname = value
+      end
+    end
+
+    def datetime_format=(datetime_format)
+      super
+      @loggers.each do |logger|
+        logger.datetime_format = datetime_format
+      end
+    end
+
+    def formatter=(formatter)
+      super
+      @loggers.each do |logger|
+        logger.formatter ||= formatter
+      end
+    end
+
+    def silence(temporary_level = ::Logger::ERROR, &block)
+      silenceable_loggers = @loggers.select do |logger|
+        logger.respond_to?(:silence)
+      end
+
+      silence_loggers(temporary_level, silenceable_loggers, &block)
+    end
+
+    def silence_loggers(temporary_level = ::Logger::ERROR, silenceable_loggers, &block)
+      return yield(self) if silenceable_loggers.empty?
+
+      silenceable_loggers.shift.silence(temporary_level) do
+        silence_loggers(temporary_level, silenceable_loggers, &block)
+      end
+    end
+    private :silence_loggers
+
+    def tagged(*tags, &block)
+      taggable_loggers = @loggers.select do |logger|
+        logger.respond_to?(:tagged)
+      end
+
+      tag_loggers(tags, taggable_loggers, &block)
+    end
+
+    def tag_loggers(tags, taggable_loggers, &block)
+      return yield(self) if taggable_loggers.empty?
+
+      taggable_loggers.shift.tagged(*tags) do
+        tag_loggers(tags, taggable_loggers, &block)
+      end
+    end
+    private :tag_loggers
+
+    # Any method not defined on standard Logger class, just send it on to anyone who will listen
+    def method_missing(name, *args, &block)
+      @loggers.each do |logger|
+        if logger.respond_to?(name)
+          logger.send(name, args, &block)
+        end
+      end
+    end
+
+    #
+    # === Synopsis
+    #
+    #   MultiLogger.new([logger1, logger2])
+    #
+    # === Args
+    #
+    # +loggers+::
+    #   An array of loggers. Each one gets every message that is sent to the MultiLogger instance
+    #
+    # === Description
+    #
+    # Create an instance.
+    #
+    def initialize(loggers)
+      @loggers = []
+      super(nil)
+      @loggers = Array(loggers)
+    end
+
+    # Methods that write to logs just write to each contained logger in turn
+    def add(severity, message = nil, progname = nil, &block)
+      @loggers.each do |logger|
+        logger.add(severity, message, progname, &block)
+      end
+    end
+
+    def <<(msg)
+      @loggers.each do |logger|
+        logger << msg
+      end
+    end
+
+    def debug(progname = nil, &block)
+      @loggers.each do |logger|
+        logger.debug(progname, &block)
+      end
+    end
+
+    def info(progname = nil, &block)
+      @loggers.each do |logger|
+        logger.info(progname, &block)
+      end
+    end
+
+    def warn(progname = nil, &block)
+      @loggers.each do |logger|
+        logger.warn(progname, &block)
+      end
+    end
+
+    def error(progname = nil, &block)
+      @loggers.each do |logger|
+        logger.error(progname, &block)
+      end
+    end
+
+    def fatal(progname = nil, &block)
+      @loggers.each do |logger|
+        logger.fatal(progname, &block)
+      end
+    end
+
+    def unknown(progname = nil, &block)
+      @loggers.each do |logger|
+        logger.unknown(progname, &block)
+      end
+    end
+
+    def close
+      @loggers.each do |logger|
+        logger.close if logger.respond_to?(:close)
+      end
+    end
+  end
+end

data/lib/logstash-logger/railtie.rb

@@ -0,0 +1,51 @@
+require 'rails/railtie'
+
+module LogStashLogger
+  def self.setup(app)
+    return unless app.config.logstash.present?
+
+    logger_options = app.config.logstash
+
+    sanitized_logger_options = if logger_options.is_a?(Array)
+                                 logger_options.map do |opts|
+                                   sanitize_logger_options(app, opts)
+                                 end
+                               else
+                                 sanitize_logger_options(app, logger_options)
+                               end
+
+    logger = LogStashLogger.new(sanitized_logger_options)
+
+    logger.level = ::Logger.const_get(app.config.log_level.to_s.upcase)
+
+    app.config.logger = logger
+  end
+
+  def self.sanitize_logger_options(app, logger_options)
+    # Convert logger options to OrderedOptions if regular Hash
+    logger_options = ActiveSupport::OrderedOptions.new.merge(logger_options)
+
+    if parsed_uri_options = LogStashLogger::Device.parse_uri_config(logger_options)
+      logger_options.delete(:uri)
+      logger_options.merge!(parsed_uri_options)
+    end
+
+    if logger_options.type == :file
+      logger_options.path ||= app.config.paths["log"].first
+    end
+
+    if app.config.respond_to?(:autoflush_log)
+      logger_options.sync = app.config.autoflush_log
+    end
+
+    logger_options
+  end
+
+  class Railtie < ::Rails::Railtie
+    config.logstash = ActiveSupport::OrderedOptions.new
+
+    initializer :logstash_logger, before: :initialize_logger do |app|
+      LogStashLogger.setup(app)
+    end
+  end
+end

data/lib/logstash-logger/silenced_logging.rb

@@ -0,0 +1,83 @@
+# Adapted from:
+# https://github.com/rails/activerecord-session_store/blob/master/lib/active_record/session_store/extension/logger_silencer.rb
+# https://github.com/rails/rails/pull/16885
+
+require 'thread'
+
+# Add support for Rails-style logger silencing. Thread-safe and no dependencies.
+#
+# Setup:
+#   logger = Logger.new(STDOUT)
+#   logger.extend(LogStashLogger::SilencedLogging)
+#
+# Usage:
+#
+#   logger.silence do
+#     ...
+#   end
+#
+module LogStashLogger
+  module SilencedLogging
+    def self.extended(logger)
+      class << logger
+        attr_accessor :silencer
+        alias_method :level_without_thread_safety, :level
+        alias_method :level, :level_with_thread_safety
+        alias_method :add_without_thread_safety, :add
+        alias_method :add, :add_with_thread_safety
+
+        Logger::Severity.constants.each do |severity|
+          instance_eval <<-EOT, __FILE__, __LINE__ + 1
+            def #{severity.downcase}?                # def debug?
+              Logger::#{severity} >= level           #   DEBUG >= level
+            end                                      # end
+          EOT
+        end
+      end
+
+      logger.instance_eval do
+        self.silencer = true
+      end
+    end
+
+    def thread_level
+      Thread.current[thread_hash_level_key]
+    end
+
+    def thread_level=(level)
+      Thread.current[thread_hash_level_key] = level
+    end
+
+    def level_with_thread_safety
+      thread_level || level_without_thread_safety
+    end
+
+    def add_with_thread_safety(severity, message = nil, progname = nil, &block)
+      if (defined?(@logdev) && @logdev.nil?) || (severity || UNKNOWN) < level
+        true
+      else
+        add_without_thread_safety(severity, message, progname, &block)
+      end
+    end
+
+    # Silences the logger for the duration of the block.
+    def silence(temporary_level = Logger::ERROR)
+      if silencer
+        begin
+          self.thread_level = temporary_level
+          yield self
+        ensure
+          self.thread_level = nil
+        end
+      else
+        yield self
+      end
+    end
+
+    private
+
+    def thread_hash_level_key
+      @thread_hash_level_key ||= :"ThreadSafeLogger##{object_id}@level"
+    end
+  end
+end

data/lib/logstash-logger/tagged_logging.rb

@@ -0,0 +1,40 @@
+module LogStashLogger
+  # Shamelessly copied from ActiveSupport::TaggedLogging
+  module TaggedLogging
+    def tagged(*tags)
+      formatter.tagged(*tags) { yield self }
+    end
+
+    def flush
+      formatter.clear_tags!
+      super if defined?(super)
+    end
+
+    module Formatter
+      def tagged(*tags)
+        new_tags = push_tags(*tags)
+        yield self
+      ensure
+        pop_tags(new_tags.size)
+      end
+
+      def push_tags(*tags)
+        tags.flatten.reject{ |t| t.nil? || t.empty? }.tap do |new_tags|
+          current_tags.concat new_tags
+        end
+      end
+
+      def pop_tags(size = 1)
+        current_tags.pop size
+      end
+
+      def clear_tags!
+        current_tags.clear
+      end
+
+      def current_tags
+        Thread.current[:logstash_logger_tags] ||= []
+      end
+    end
+  end
+end

data/lib/logstash-logger/version.rb

@@ -0,0 +1,3 @@
+module LogStashLogger
+  VERSION = "0.26.1"
+end

data/lib/logstash-logger.rb

@@ -0,0 +1,11 @@
+require 'logstash-logger/version'
+
+require 'logstash/event'
+
+require 'logstash-logger/device'
+
+require 'logstash-logger/logger'
+require 'logstash-logger/formatter'
+require 'logstash-logger/configuration'
+
+require 'logstash-logger/railtie' if defined?(Rails::Railtie)

data/logstash-logger.gemspec

@@ -0,0 +1,39 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'logstash-logger/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "logstash-logger-p"
+  gem.version       = LogStashLogger::VERSION
+  gem.authors       = ["David Butler"]
+  gem.email         = ["dwbutler@ucla.edu"]
+  gem.description   = %q{Ruby logger that writes directly to LogStash}
+  gem.summary       = %q{LogStash Logger for ruby}
+  gem.homepage      = "http://github.com/dwbutler/logstash-logger"
+  gem.license       = "MIT"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency 'logstash-event', '~> 1.2'
+
+  gem.add_development_dependency 'rails'
+  gem.add_development_dependency 'redis'
+  gem.add_development_dependency 'poseidon'
+  gem.add_development_dependency 'aws-sdk-kinesis'
+  gem.add_development_dependency 'aws-sdk-firehose'
+
+  if defined?(JRUBY_VERSION)
+    gem.add_development_dependency 'SyslogLogger'
+  end
+
+  gem.add_development_dependency 'rspec', '>= 3'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'pry'
+  gem.add_development_dependency 'wwtd'
+  gem.add_development_dependency 'appraisal'
+  gem.add_development_dependency 'rubocop'
+end