sawmill 0.0.1

data/lib/sawmill/logger.rb

@@ -0,0 +1,373 @@
+# -----------------------------------------------------------------------------
+# 
+# Sawmill logger class
+# 
+# -----------------------------------------------------------------------------
+# Copyright 2009 Daniel Azuma
+# 
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of the copyright holder, nor the names of any other
+#   contributors to this software, may be used to endorse or promote products
+#   derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+;
+
+
+if RUBY_VERSION >= '1.9'
+  require 'securerandom'
+end
+
+
+module Sawmill
+  
+  
+  # This is the Sawmill logger.
+  # It duck-types most of the API of the logger class from the ruby
+  # standard library, and adds capabilities specific to Sawmill.
+  
+  class Logger
+    
+    
+    # Create a new logger.
+    # 
+    # Supported options include:
+    # 
+    # <tt>:levels</tt>::
+    #   Use a custom Sawmill::LevelGroup. Normally, you should leave this
+    #   set to the default, which is Sawmill::STANDARD_LEVELS.
+    # <tt>:level</tt>::
+    #   Default level to use for log messages when no level is explicitly
+    #   provided. By default, this is set to the level group's default,
+    #   which in the case of the standard levels is :INFO.
+    # <tt>:attribute_level</tt>::
+    #   Default level to use for attributes when no level is explicitly
+    #   provided. By default, this is set to the level group's highest,
+    #   level, which in the case of the standard levels is :ANY.
+    # <tt>:progname</tt>::
+    #   Progname to use in log messages. Default is "sawmill".
+    # <tt>:record_progname</tt>::
+    #   Progname to use in special log entries dealing with log records
+    #   (i.e. record delimiters and attribute messages). Default is the
+    #   same as the normal progname setting.
+    # <tt>:record_id_generator</tt>::
+    #   A proc that generates and returns a new record ID if one is not
+    #   explicitly passed into begin_record. If you do not provide a
+    #   generator, the default one is used, which generates an ID using the
+    #   variant 4 (random) UUID standard.
+    # <tt>:processor</tt>::
+    #   A processor for log entries generated by this logger.
+    #   If not specified, log entries are written out to STDOUT.
+    
+    def initialize(opts_={})
+      @levels = opts_[:levels] || ::Sawmill::STANDARD_LEVELS
+      @level = @levels.get(opts_[:level])
+      if opts_.include?(:attribute_level)
+        @attribute_level = @levels.get(opts_[:attribute_level])
+      else
+        @attribute_level = @levels.highest
+      end
+      @progname = opts_[:progname] || 'sawmill'
+      @record_progname = opts_[:record_progname] || @progname
+      @record_id_generator = opts_[:record_id_generator] || _get_default_record_id_generator
+      @processor = opts_[:processor] || ::Sawmill::Formatter.new(STDOUT)
+      @current_record_id = nil
+    end
+    
+    
+    # Emit a log message. This method has the same behavior as the
+    # corresponding method in ruby's logger class.
+    
+    def add(level_, message_=nil, progname_=nil, &block_)
+      level_obj_ = @levels.get(level_)
+      if level_obj_.nil?
+        raise Errors::UnknownLevelError, level_
+      end
+      return true if level_obj_ < @level
+      progname_ ||= @progname
+      if message_.nil?
+        if block_given?
+          message_ = yield
+        else
+          message_ = progname_
+          progname_ = @progname
+        end
+      end
+      case message_
+      when ::String
+        # Do nothing
+      when ::Exception
+        message_ = "#{message_.message} (#{message_.class})\n" +
+          (message_.backtrace || []).join("\n")
+      else
+        message_ = message_.inspect
+      end
+      @processor.message(Entry::Message.new(level_obj_, Time.now, progname_, @current_record_id, message_))
+      true
+    end
+    alias_method :log, :add
+    
+    
+    def to_s  # :nodoc:
+      inspect
+    end
+    
+    def inspect  # :nodoc:
+      "#<#{self.class}:0x#{object_id.to_s(16)} progname=#{@progname.inspect} level=#{@level.name}>"
+    end
+    
+    
+    # Emits an "unknown" log entry. This is equivalent to the corresponding
+    # method in ruby's logger class, which dumps the given string to the log
+    # device without any formatting. Normally, you would not use this method
+    # because it bypasses the log formatting and parsing capability.
+    
+    def <<(message_)
+      add(@levels.default, message_)
+    end
+    
+    
+    # Emits a begin_record log entry. This begins a new log record.
+    # 
+    # If you pass a string ID, that ID is used as the record ID for the new
+    # log record. If you leave it as nil, an ID is generated for you, using
+    # the record id generator for this logger. In either case, the record ID
+    # for the new record is returned.
+    # 
+    # If you call this when a record is already open, the current record is
+    # automatically closed before the new record is opened. That is, an
+    # end_record is implicitly called in this case.
+    
+    def begin_record(id_=nil)
+      end_record if @current_record_id
+      @current_record_id = (id_ || @record_id_generator.call).to_s
+      @processor.begin_record(Entry::BeginRecord.new(@levels.highest, Time.now, @record_progname, @current_record_id))
+      @current_record_id
+    end
+    
+    
+    # Returns the record ID for the currently open log record, or nil if
+    # there is not a log record currently open.
+    
+    def current_record_id
+      @current_record_id
+    end
+    
+    
+    # Ends the current log record by emitting an end_record log entry, if
+    # a record is currently open. Returns the record ID of the ended log
+    # record if one was open, or nil if no log record was open.
+    
+    def end_record
+      if @current_record_id
+        @processor.end_record(Entry::EndRecord.new(@levels.highest, Time.now, @record_progname, @current_record_id))
+        id_ = @current_record_id
+        @current_record_id = nil
+        id_
+      else
+        nil
+      end
+    end
+    
+    
+    # Emits an attribute log entry in the current record.
+    # You must specify a key and a value as strings, and an operation.
+    # The operation defaults to <tt>:set</tt> if not specified.
+    # 
+    # If you specify a level, it will be used; otherwise the logger's
+    # default attribute level is used.
+    # Raises Errors::UnknownLevelError if you specify a level that doesn't
+    # exist.
+    
+    def attribute(key_, value_, operation_=nil, level_=true, progname_=nil)
+      if level_ == true
+        level_obj_ = @attribute_level
+      else
+        level_obj_ = @levels.get(level_)
+        if level_obj_.nil?
+          raise Errors::UnknownLevelError, level_
+        end
+      end
+      return true if level_obj_ < @level
+      @processor.attribute(Entry::Attribute.new(level_obj_, Time.now, progname_ || @record_progname, @current_record_id, key_, value_, operation_))
+      true
+    end
+    
+    
+    # Emits a set-attribute log entry in the current record.
+    # You must specify a key and a value as strings.
+
+    
+    def set_attribute(key_, value_)
+      attribute(key_, value_, :set)
+    end
+    
+    
+    # Emits an append-attribute log entry in the current record.
+    # You must specify a key and a value as strings.
+
+    
+    def append_attribute(key_, value_)
+      attribute(key_, value_, :append)
+    end
+    
+    
+    # Close the logger by closing the log entry processor to which it is
+    # emitting log entries.
+    
+    def close
+      @processor.close
+    end
+    
+    
+    # Get the current progname setting for this logger
+    
+    def progname
+      @progname
+    end
+    
+    
+    # Set the current progname setting for this logger
+    
+    def progname=(value_)
+      @progname = value_.to_s.gsub(/\s+/, '')
+    end
+    
+    
+    # Get the current level setting for this logger as a Sawmill::Level.
+    
+    def level
+      @level
+    end
+    
+    
+    # Set the current level setting for this logger.
+    # You may specify the level as a string, a symbol, an integer, or a
+    # Sawmill::Level. Ruby's logger constants such as ::Logger::INFO
+    # will also work.
+    
+    def level=(value_)
+      if value_.kind_of?(Level)
+        @level = value_
+      else
+        level_obj_ = @levels.get(value_)
+        if level_obj_.nil?
+          raise Errors::UnknownLevelError, value_
+        end
+        @level = level_obj_
+      end
+    end
+    
+    alias_method :sev_threshold=, :level=
+    alias_method :sev_threshold, :level
+    
+    
+    # Provide a block that generates and returns a unique record ID string.
+    # This block will be called when begin_record is called without an
+    # explicit ID provided. If you do not provide a block, Sawmill will use
+    # a default generator which uses the variant 4 (random) UUID standard.
+    
+    def to_generate_record_id(&block_)
+      @record_id_generator = block_ || _get_default_record_id_generator
+    end
+    
+    
+    # You may call additional methods on the logger as shortcuts to log
+    # messages at specific levels, or to query whether the logger is logging
+    # to a given level. These methods match the corresponding methods in the
+    # classic ruby logger object, except that they are configurable for
+    # custom level schemes.
+    # 
+    # For example, in the standard level scheme, the method "info" is
+    # defined, so you may call:
+    # 
+    #   logger.info("MainApp") { "Received connection from #{ip}" }
+    #   # ...
+    #   logger.info "Waiting for input from user"
+    #   # ...
+    #   logger.info { "User typed #{input}" }
+    # 
+    # You may also call:
+    # 
+    #   logger.info?  # Returns true if INFO messages are accepted
+    # 
+    # Methods available in the standard level scheme are as follows:
+    # 
+    # * <tt>debug</tt>
+    # * <tt>info</tt>
+    # * <tt>warn</tt>
+    # * <tt>error</tt>
+    # * <tt>fatal</tt>
+    # * <tt>unknown</tt>
+    # * <tt>any</tt>
+    # 
+    # The "unknown" and "any" methods both correspond to the +ANY+ level.
+    # The latter is the preferred name under Sawmill. The former is for
+    # backward compatibility with ruby's classic logger.
+    
+    def method_missing(method_, *args_, &block_)
+      method_name_ = method_.to_s
+      question_ = method_name_[-1..-1] == '?'
+      method_name_ = method_name_[0..-2] if question_
+      level_ = @levels.lookup_method(method_name_)
+      return super(method_, *args_, &block_) unless level_
+      if question_
+        level_ >= @level
+      else
+        add(level_, nil, args_[0], &block_)
+      end
+    end
+    
+    
+    def _get_default_record_id_generator  # :nodoc:
+      if RUBY_VERSION >= '1.9'
+        lambda do
+          uuid_ = SecureRandom.hex(32)
+          uuid_[12] = '4'
+          uuid_[16] = (uuid_[16,1].to_i(16)&3|8).to_s(16)
+          uuid_.insert(8, '-')
+          uuid_.insert(13, '-')
+          uuid_.insert(18, '-')
+          uuid_.insert(23, '-')
+          uuid_
+        end
+      else
+        lambda do
+          uuid_ = Kernel.rand(0x100000000000000000000000000000000).to_s(16).rjust(32, '0')
+          uuid_[12] = '4'
+          uuid_[16] = (uuid_[16,1].to_i(16)&3|8).to_s(16)
+          uuid_.insert(8, '-')
+          uuid_.insert(13, '-')
+          uuid_.insert(18, '-')
+          uuid_.insert(23, '-')
+          uuid_
+        end
+      end
+    end
+    private :_get_default_record_id_generator
+    
+    
+  end
+  
+  
+end

data/lib/sawmill/parser.rb

@@ -0,0 +1,181 @@
+# -----------------------------------------------------------------------------
+# 
+# Sawmill stream parser utility
+# 
+# -----------------------------------------------------------------------------
+# Copyright 2009 Daniel Azuma
+# 
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of the copyright holder, nor the names of any other
+#   contributors to this software, may be used to endorse or promote products
+#   derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+;
+
+
+module Sawmill
+  
+  
+  # A logfile parser that parses log entries from a logfile and sends them
+  # to an entry processor.
+  
+  class Parser
+    
+    # :stopdoc:
+    LINE_REGEXP = /^\[\s*([[:graph:]]+)\s+(\d{4})-(\d{2})-(\d{2})(T|\s)(\d{2}):(\d{2}):(\d{2})(.(\d{1,6}))?Z?\s?([+-]\d{4})?\s+([[:graph:]]+)(\s+([[:graph:]]+))?\s+([\^$.=])\]\s(.*)$/
+    DIRECTIVE_REGEXP = /^#\s+sawmill_format:\s+(\w+)=(.*)$/
+    ATTRIBUTE_REGEXP = /^([[:graph:]]+)\s([=+\/-])\s/
+    # :startdoc:
+    
+    
+    # Create a new parser that reads from the given stream.
+    # 
+    # You should provide a processor to receive the data from the logfile.
+    # The processor may be either an entry processor or a record processor.
+    # You may also pass nil for the processor. In this case, the generated
+    # log entries will not be sent to a processor but will still be returned
+    # by the parse_one_entry method.
+    # 
+    # Recognized options include:
+    # 
+    # <tt>:levels</tt>
+    #   Sawmill::LevelGroup to use to parse log levels.
+    #   If not specified, Sawmill::STANDARD_LEVELS is used by default.
+    # <tt>:emit_incomplete_records_on_complete</tt>
+    #   If set to true, causes any incomplete log records to be emitted
+    #   in their incomplete state when EOF is reached.
+    
+    def initialize(io_, processor_, opts_={})
+      @io = io_
+      @record_processor = nil
+      @processor = nil
+      if processor_.respond_to?(:record) && processor_.respond_to?(:extra_entry)
+        @processor = RecordBuilder.new(processor_)
+      elsif processor_.respond_to?(:begin_record) && processor_.respond_to?(:end_record)
+        @processor = processor_
+      end
+      @levels = opts_[:levels] || ::Sawmill::STANDARD_LEVELS
+      @emit_incomplete_records_on_complete = opts_[:emit_incomplete_records_on_complete]
+      @current_record_id = nil
+      @parser_directives = {}
+    end
+    
+    
+    # Parse one log entry from the stream and emit it to the processor.
+    # Also returns the log entry.
+    # Returns nil if EOF has been reached.
+    
+    def parse_one_entry
+      str_ = @io.gets
+      entry_ = nil
+      if str_
+        match_ = LINE_REGEXP.match(str_)
+        if match_
+          level_ = @levels.get(match_[1])
+          timestamp_ = ::Time.utc(match_[2].to_i, match_[3].to_i, match_[4].to_i,
+            match_[6].to_i, match_[7].to_i, match_[8].to_i, match_[10].to_s.ljust(6, '0').to_i)
+          offset_ = match_[11].to_i
+          if offset_ != 0
+            neg_ = offset_ < 0
+            offset_ = -offset_ if neg_
+            secs_ = offset_ / 100 * 3600 + offset_ % 100 * 60
+            if neg_
+              timestamp_ += secs_
+            else
+              timestamp_ -= secs_
+            end
+          end
+          progname_ = match_[12]
+          record_id_ = match_[14] || @current_record_id
+          type_code_ = match_[15]
+          str_ = match_[16]
+          if str_ =~ /(\\+)$/
+            count_ = $1.length
+            str_ = $` + "\\"*(count_/2)
+            while count_ % 2 == 1
+              str2_ = @io.gets
+              if str2_ && str2_ =~ /(\\*)\n?$/
+                count_ = $1.length
+                str_ << "\n" << $` << "\\"*(count_/2)
+              else
+                break
+              end
+            end
+          end
+          case type_code_
+          when '^'
+            if str_ =~ /^BEGIN\s/
+              @current_record_id = $'
+              entry_ = Entry::BeginRecord.new(level_, timestamp_, progname_, @current_record_id)
+              @processor.begin_record(entry_) if @processor
+            end
+          when '$'
+            if str_ =~ /^END\s/
+              @current_record_id = $'
+              entry_ = Entry::EndRecord.new(level_, timestamp_, progname_, @current_record_id)
+              @current_record_id = nil
+              @processor.end_record(entry_) if @processor
+            end
+          when '='
+            if str_ =~ ATTRIBUTE_REGEXP
+              key_ = $1
+              opcode_ = $2
+              value_ = $'
+              operation_ = opcode_ == '+' ? :append : :set
+              entry_ = Entry::Attribute.new(level_, timestamp_, progname_, record_id_, key_, value_, operation_)
+              @processor.attribute(entry_) if @processor
+            end
+          end
+          unless entry_
+            entry_ = Entry::Message.new(level_, timestamp_, progname_, record_id_, str_)
+            @processor.message(entry_) if @processor
+          end
+        else
+          if str_ =~ DIRECTIVE_REGEXP
+            @parser_directives[$1] = $2
+          end
+          entry_ = Entry::UnknownData.new(str_)
+          @processor.unknown_data(entry_) if @processor.respond_to?(:unknown_data)
+        end
+      else
+        if @emit_incomplete_records_on_complete && @processor.respond_to?(:emit_incomplete_records)
+          @processor.emit_incomplete_records
+        end
+      end
+      entry_
+    end
+    
+    
+    # Parse the rest of the stream until EOF is reached, and emit the log
+    # entries to the processor.
+    
+    def parse_all
+      while process_one_entry; end
+    end
+    
+    
+  end
+  
+  
+end