semantic_logger 0.0.2

data/Rakefile

@@ -0,0 +1,37 @@
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+
+require 'rubygems'
+require 'rake/clean'
+require 'rake/testtask'
+require 'date'
+require 'semantic_logger/version'
+
+desc "Build gem"
+task :gem  do |t|
+  gemspec = Gem::Specification.new do |spec|
+    spec.name        = 'semantic_logger'
+    spec.version     = SemanticLogger::VERSION
+    spec.platform    = Gem::Platform::RUBY
+    spec.authors     = ['Reid Morrison']
+    spec.email       = ['reidmo@gmail.com']
+    spec.homepage    = 'https://github.com/ClarityServices/semantic_logger'
+    spec.date        = Date.today.to_s
+    spec.summary     = "Semantic Logger for Ruby, and Ruby on Rails"
+    spec.description = "Logging with additional machine readable and parseable log data"
+    spec.files       = FileList["./**/*"].exclude('*.gem', 'nbproject').map{|f| f.sub(/^\.\//, '')}
+    spec.has_rdoc    = true
+    spec.add_development_dependency 'shoulda'
+  end
+  Gem::Builder.new(gemspec).build
+end
+
+desc "Run Test Suite"
+task :test do
+  Rake::TestTask.new(:functional) do |t|
+    t.test_files = FileList['test/*_test.rb']
+    t.verbose    = true
+  end
+
+  Rake::Task['functional'].invoke
+end

data/init.txt

@@ -0,0 +1,59 @@
+--  if (($0 == 'irb' || $0 == 'jirb') && ENV['RAILS_CONSOLE_STDOUT']) || ENV['RAILS_LOGGER_STDOUT']
+--    config.logger = Logger.new(STDOUT)
+--  end
+++  # Setup Semantic Logging
+++#  require 'lib/semantic_logger'
+++#  if (($0 == 'irb' || $0 == 'jirb') && ENV['RAILS_CONSOLE_STDOUT']) || ENV['RAILS_LOGGER_STDOUT']
+++#    config.logger = SemanticLogger::LoggerAppender.new(Logger.new(STDOUT))
+++#  else
+++#    exc = nil
+++#    begin
+++#      # Don't use Mongo Log Appender in dev and test
+++#      unless false #Rails.env.test? #or Rails.env.development?
+++#        require 'mongo'
+++#        require 'lib/auto_failover_repl_set_connection'
+++#        # Log only to Mongo
+++#        cfg = YAML.load_file(File.join(Rails.root, "config", "mongo.yml"))[Rails.env]
+++#        conn = Mongo::AutoFailoverReplSetConnection.from_uri(cfg['uri'],cfg['options'] || {})
+++#        SemanticLogger::MongoAppender.db = conn.db("logger_#{Rails.env}")
+++#        SemanticLogger::MongoAppender.create_indexes
+++#        config.logger = SemanticLogger::MongoAppender.new('Rails', config.log_level)
+++#      end
+++#    rescue StandardError => exc
+++#    end
+++#
+++#    begin
+++#      unless config.logger
+++#        # Wrap the BufferedLogger with the Semantic Logger
+++#        logger = ActiveSupport::BufferedLogger.new(config.log_path)
+++#        logger.level = ActiveSupport::BufferedLogger.const_get(config.log_level.to_s.upcase)
+++#        logger.auto_flushing = false if config.environment == "production"
+++#        config.logger = SemanticLogger::LoggerAppender.new(logger)
+++#        config.logger.error("Failed to start Mongo Logger, falling back to File Logging", exc) if exc
+++#      end
+++#    rescue StandardError => exc
+++#    end
+++#
+++#    unless config.logger
+++#      config.logger = ActiveSupport::BufferedLogger.new(STDERR)
+++#      config.logger.level = ActiveSupport::BufferedLogger::WARN
+++#      config.logger.error("Failed to start SemanticLogger::LoggerAppender")
+++#      config.logger.error(exc) if exc
+++#      config.logger.warn(
+++#          "Rails Error: Unable to access log file. Please ensure that #{config.log_path} exists and is chmod 0666. " +
+++#          "The log level has been raised to WARN and the output directed to STDERR until the problem is fixed."
+++#      )
+++#    end
+++
+++    # Rails.logger =
+++#    def initialize_framework_logging
+++#      for framework in ([ :active_record, :action_controller, :action_mailer ] & configuration.frameworks)
+++#        framework.to_s.camelize.constantize.const_get("Base").logger ||= Rails.logger
+++#      end
+++#
+++#      ActiveSupport::Dependencies.logger ||= Rails.logger
+++#      Rails.cache.logger ||= Rails.logger
+++#    end
+++
+++
+++#  end

data/lib/semantic_logger/appender/logger.rb

@@ -0,0 +1,102 @@
+# Logger appender
+#
+#   Maps the SemanticLogger API's to the Rails log, log4j, or Ruby Logger
+#
+# Installation:
+#    Rails.logger = SemanticLogger::Appender::Logger.new(Rails.logger)
+#
+# Also works with the Ruby Logger
+#    require 'logger'
+#    require 'semantic_logger'
+#    logger = Logger.new(STDOUT)
+#    Rails.log = SemanticLogger::Appender::Logger.new(logger)
+#
+# ActiveResource::BufferedLogger
+#    ...
+#
+# The log level is controlled by the Logging implementation passed into
+# this appender
+module SemanticLogger
+  module Appender
+    class Logger
+      attr_reader :logger
+
+      def initialize(logger)
+        raise "logger cannot be null when initiailizing the SemanticLogging::Appender::Logger" unless logger
+        @logger = logger
+      end
+
+      # The default log formatter
+      # Generates logs of the form:
+      #    2011-07-19 14:36:15.660 D [1149:ScriptThreadProcess] Rails -- Hello World
+      @@formatter = Proc.new do |level, name, message, time, duration|
+        str = "#{time.strftime("%Y-%m-%d %H:%M:%S")}.#{"%03d" % (time.usec/1000)} #{level.to_s[0..0].upcase} [#{$$}:#{thread_name}] #{name} -- #{message}\n"
+        str << " (#{duration}ms)" if duration
+        str
+      end
+
+      # For JRuby include the Thread name rather than its id
+      if defined? Java
+        def self.thread_name
+          Java::java.lang::Thread.current_thread.name
+        end
+      else
+        def self.thread_name
+          Thread.object_id
+        end
+      end
+
+      # Allow the global formatter to be replaced
+      def self.formatter=(formatter)
+        @@formatter = formatter
+      end
+
+      # Pass log calls to the underlying Rails, log4j or Ruby logger
+      #  trace entries are mapped to debug since :trace is not supported by the
+      #  Ruby or Rails Loggers
+      def log(level, name, message, hash, &block)
+        @logger.send(level == :trace ? :debug : level) { self.class.format_message(level, name, message, hash, &block) }
+      end
+
+      # Convert a semantic log entry into plain text
+      def self.format_message(level, name, message, hash=nil, &block)
+        # TODO need to define :try if not already defined. E.g. Outside Rails
+        msg = time = duration = nil
+        if hash
+          msg      = hash.delete(:message)
+          time     = hash.delete(:time)
+          duration = hash.delete(:duration)
+        end
+        msg  ||= message.to_s
+        time ||= Time.now
+
+        msg << " -- " << self.msg2str(hash) if hash
+        msg << " -- " << self.msg2str(block.call) if block
+        @@formatter.call(level, name, msg, time, duration)
+      end
+
+      # Convert Objects to Strings for text based logging
+      def self.msg2str(message)
+        case message
+        when ::String
+          message
+        when ::Exception
+          "#{message.class}: #{message.message}\n#{(message.backtrace || []).join("\n")}"
+        when ::Hash
+          # With a hash, the message can be an element of the hash itself
+          if msg = message[:message]
+            # Cannot change supplied hash
+            hash = message.clone
+            hash.delete(:message)
+            "#{msg} #{hash.inspect}"
+          else
+            message.inspect
+          end
+        else
+          message.inspect
+        end
+      end
+
+    end
+  end
+end

data/lib/semantic_logger/appender/mongodb.rb

@@ -0,0 +1,219 @@
+module SemanticLogger
+  module Appender
+    # The Mongo Appender for the SemanticLogger
+    # clarity_log Schema:
+    #    _id        : ObjectId("4d9cbcbf7abb3abdaf9679cd"),
+    #    time       : ISODate("2011-04-06T19:19:27.006Z"),
+    #    host_name  : "Name of the host on which this log entry originated",
+    #    application: 'Name of application or service logging the data - clarity_base, nginx, tomcat',
+    #    pid        : process id
+    #    thread     : "name of thread",
+    #    name       : "ch.qos.logback.classic.db.mongo.MongoDBAppenderTest",
+    #    level      : 'trace|debug|warn|info|error',
+    #    message    : "blah blah",
+    #    duration   : ms,  # Set by Logger#benchmark
+    #    tracking_number : "Some tracking id"
+    #    metadata : {
+    #      Optional. Any user supplied data, including any thread specific context variables
+    #      values supplied on a per log entry will override any thread context values
+    #    }
+    #    # When an exception is supplied as the first or second parameter
+    #    # If supplied as the first parameter, message='exception name'
+    #    exception: {
+    #      name: 'MyException',
+    #      description: 'blah',
+    #      stack_trace: []
+    #    }
+    #
+    #    # For trace and debug level logging, the following can also be logged
+    #    # for all levels. Not on for higher levels due to performance impact
+    #    source: {
+    #      filename:
+    #      method:
+    #      line_number:
+    #    }
+    #
+    # # Future, the Rails around filter can log the following additional data
+    #   controller:
+    #   action:
+    #   duration: 'ms'
+    #   http_verb: 'get|post|..'
+    #   params: Hash
+    #
+    #   tracking_number: 'user defined tracking number'
+    class MongoDB
+      attr_reader :db, :collection_name
+      attr_accessor :formatter, :host_name, :safe, :application
+
+      # Create a MongoDB Appender instance
+      # SemanticLogger::Appender::MongoDB.new(
+      #         :db => Cache::Work.db
+      #        )
+      #
+      # Parameters:
+      # :db [Mongo::Database]
+      #   The MongoDB database connection to use, not the database name
+      #
+      # :collection_name [String]
+      #   Name of the collection to store log data in
+      #   Default: semantic_logger
+      #
+      # :host_name [String]
+      #   host_name to include in the document logged to Mongo
+      #   Default: first part of host name extracted from Socket
+      #
+      # :safe [Boolean]
+      #   Whether to use safe write for logging
+      #   Not recommended to change this value except to diagnose connection
+      #   issues or when log entries are not being written to Mongo
+      #   Default: false
+      #
+      # :application [String]
+      #   Name of the application to include in the document written to mongo
+      #   Default: nil (None)
+      #
+      # :collection_size [Integer]
+      #   The size of the MongoDB capped collection to create in bytes
+      #   Default: 1 GB
+      #
+      #   Some examples
+      #     Prod: 25GB (.5GB per day across 4 servers over 10 days)
+      #     Dev: .5GB
+      #     Test: File
+      #     Release: 4GB
+      #
+      #  :collection_max [Integer]
+      #    Maximum number of log entries that the capped collection will hold
+      #
+      def initialize(params={}, &block)
+        @db              = params[:db] || raise('Missing mandatory parameter :db')
+        @collection_name = params[:collection_name] || 'semantic_logger'
+        @host_name       = params[:host_name] || Socket.gethostname.split('.').first
+        @safe            = params[:safe] || false
+        @application     = params[:application]
+
+        # Create a collection that will hold the lesser of 1GB space or 10K documents
+        @collection_size = params[:collection_size] || 1024**3
+        @collection_max  = params[:collection_max]
+
+        # Set the formatter to the supplied block
+        @formatter       = block || self.default_formatter
+
+        # Create the collection and necessary indexes
+        create_indexes
+      end
+
+      # Create the required capped collection
+      # Features of capped collection:
+      # * No indexes by default (not even on _id)
+      # * Documents cannot be deleted,
+      # * Document updates cannot make them any larger
+      # * Documents are always stored in insertion order
+      #   * A find will always return the documents in their insertion order
+      def create_indexes
+        db.create_collection(collection_name, {:capped => true, :size => @collection_size, :max => @collection_max})
+      end
+
+      # Purge all data from the capped collection by dropping the collection
+      # and recreating it.
+      # Also useful when the size of the capped collection needs to be changed
+      def purge_all
+        collection.drop
+        @collection = nil
+        create_indexes
+      end
+
+      # Return the collection being used to write the log document to
+      def collection
+        @collection ||= db[collection_name]
+      end
+
+      # For JRuby include the Thread name rather than its id
+      if defined? Java
+        def self.thread_name
+          Java::java.lang::Thread.current_thread.name
+        end
+      else
+        def self.thread_name
+          Thread.object_id
+        end
+      end
+
+      # Default log formatter
+      #  Replace this formatter by supplying a Block to the initializer
+      def default_formatter
+        Proc.new do |level, name, message, hash, block|
+          document = {
+            :time      => Time.now,
+            :host_name => host_name,
+            :pid       => $PID,
+            :thread    => SemanticLogger::Appender::MongoDB.thread_name,
+            :name      => name,
+            :level     => level,
+          }
+          document[:application] = application if application
+          document[:message] = self.class.strip_colorizing(message) if message
+
+          SemanticLogger::Appender::MongoDB.populate(document, hash) if hash
+          SemanticLogger::Appender::MongoDB.populate(document, block.call) if block
+          document
+        end
+      end
+
+      # Strip the standard Rails colorizing from the logged message
+      def self.strip_colorizing(message)
+        message.to_s.gsub(/(\e(\[([\d;]*[mz]?))?)?/, '').strip
+      end
+
+      # Log the message
+      def log(level, name, message, hash, &block)
+        # Insert log entry into Mongo
+        # Use safe=>false so that we do not wait for it to be written to disk, or
+        # for the response from the MongoDB server
+        document = formatter.call(level, name, message, hash, &block)
+        collection.insert(document, :safe=>safe)
+      end
+
+      # Populate Log Hash
+      def self.populate(document, message)
+        case message
+        when ::String
+          if document[:message]
+            document[:message] << " " << strip_colorizing(message)
+          else
+            document[:message] = strip_colorizing(message)
+          end
+        when ::Exception
+          document[:exception] = {
+            :class => message.class.name,
+            :message => message.message,
+            :backtrace => message.backtrace
+          }
+        when ::Hash
+          # With a hash, the message can be an element of the hash itself
+          if msg = message[:message]
+            # Cannot change supplied hash
+            hash = message.clone
+            hash.delete(:message)
+            if document[:message]
+              document[:message] << " " << strip_colorizing(msg)
+            else
+              document[:message] = strip_colorizing(msg)
+            end
+            document[:metadata] = hash
+          else
+            document[:metadata] = message
+          end
+        else
+          if document[:message]
+            document[:message] << " " << msg.inspect
+          else
+            document[:message] = msg.inspect
+          end
+        end
+        document
+      end
+
+    end
+  end
+end

data/lib/semantic_logger/logger.rb

@@ -0,0 +1,136 @@
+# Logger is the interface used by
+#
+# Logger maintains the logging name to be used for all log entries generated
+# by the invoking classes or modules
+#
+# It is recommended to create an instance of the class for every class or
+# module so that it can be uniquely identified and searched on
+#
+# Example, log to Logger:
+#   require 'logger'
+#   require 'semantic_logger'
+#   log = Logger.new(STDOUT)
+#   log.level = Logger::DEBUG
+#
+#   SemanticLogger::Manager.register_appender(SemanticLogger::Appender::Logger.new(log))
+#
+#   logger = SemanticLogger::Logger.new("my.app.class")
+#   logger.debug("Login time", :user => 'Joe', :duration => 100, :ip_address=>'127.0.0.1')
+#
+# # Now log to the Logger above as well as Mongo at the same time
+#
+#   SemanticLogger::Manager.register_appender(SemanticLogger::Appender::Mongo.new(cfg))
+# ...
+#   logger.debug("Login time", :user => 'Mary', :duration => 230, :ip_address=>'192.168.0.1')
+module SemanticLogger
+  class Logger
+    include SyncAttr
+
+    # Logging levels in order of precendence
+    LEVELS = [:trace, :debug, :info, :warn, :error]
+
+    # Mapping of Rails and Ruby Logger levels to SemanticLogger levels
+    MAP_LEVELS = []
+    ::Logger::Severity.constants.each do |constant|
+      MAP_LEVELS[::Logger::Severity.const_get(constant)] = LEVELS.find_index(constant.downcase.to_sym) || LEVELS.find_index(:error)
+    end
+
+    # Thread safe Class Attribute accessor for appenders array
+    sync_cattr_accessor :appenders do
+      []
+    end
+
+    # Allow for setting the default log level
+    def self.default_level=(default_level)
+      @@default_level = default_level
+    end
+
+    def self.default_level
+      @@default_level
+    end
+
+    attr_reader :application, :level
+
+    @@default_level = :info
+
+    # Create a Logger instance
+    # Parameters:
+    #  application: A class, module or a string with the application/class name
+    #               to be used in the logger
+    #  options:
+    #    :level   The initial log level to start with for this logger instance
+    def initialize(application, options={})
+      @application = application.is_a?(String) ? application : application.name
+      set_level(options[:level] || self.class.default_level)
+    end
+
+    # Set the logging level
+    # Must be one of the values in #LEVELS
+    def level=(level)
+      set_level(level)
+    end
+
+    # Implement the log level calls
+    #   logger.debug(message|hash|exception, hash|exception=nil, &block)
+    #
+    # Implement the log level query
+    #   logger.debug?
+    #
+    # Example:
+    #   logger = SemanticLogging::Logger.new('MyApplication')
+    #   logger.debug("Only display this if log level is set to Debug or lower")
+    #
+    #   # Log semantic information along with a text message
+    #   logger.info("Request received", :user => "joe", :duration => 100)
+    #
+    #   # Log an exception in a semantic way
+    #   logger.info("Parsing received XML", exc)
+    #
+    LEVELS.each_with_index do |level, index|
+      class_eval <<-EOT, __FILE__, __LINE__
+        def #{level}(message = nil, data = nil, &block)                                                          # def trace(message = nil, data = nil, &block)
+          if @level_index <= #{index}                                                                            #   if @level_index <= 0
+            self.class.appenders.each {|appender| appender.log(:#{level}, application, message, data, &block) }  #     self.class.appenders.each {|appender| appender.log(:trace, application, message, data, &block) }
+            true                                                                                                 #     true
+          else                                                                                                   #   else
+            false                                                                                                #     false
+          end                                                                                                    #   end
+        end                                                                                                      # end
+
+        def #{level}?                                                                                            # def trace?
+          @level_index <= #{index}                                                                               #   @level_index <= 0
+        end                                                                                                      # end
+      EOT
+    end
+
+    # Semantic Logging does not support :fatal or :unknown levels since these
+    # are not understood by the majority of the logging providers
+    # Map them to :error
+    alias :fatal :error
+    alias :fatal? :error?
+    alias :unknown :error
+    alias :unknown? :error?
+
+    # forward other calls to ActiveResource::BufferedLogger
+    # #silence is not implemented since it is not thread safe prior to Rails 3.2
+    # #TODO implement a thread safe silence method
+
+    protected
+
+    # Verify and set the level
+    def set_level(level)
+      index = if level.is_a?(Integer)
+        MAP_LEVELS[level]
+      elsif level.is_a?(String)
+        level = level.downcase.to_sym
+        LEVELS.index(level)
+      else
+        LEVELS.index(level)
+      end
+
+      raise "Invalid level:#{level.inspect} being requested. Must be one of #{LEVELS.inspect}" unless index
+      @level_index = index
+      @level = level
+    end
+  end
+end