sawmill 0.0.1 → 0.0.2

data/History.rdoc

@@ -1,3 +1,9 @@
+=== 0.0.2 / 2009-10-28
+
+* Added multi-parser utility.
+* Fixed several bugs involving parsing and formatting.
+* Logger uses activesupport's SecureRandom on Ruby 1.8 if available.
+
 === 0.0.1 / 2009-10-28
 
 * Initial test release

data/Rakefile

@@ -80,7 +80,7 @@ gemspec_ = ::Gem::Specification.new do |s_|
   s_.author = 'Daniel Azuma'
   s_.email = 'dazuma@gmail.com'
   s_.description = 'Sawmill is a logging and log analysis system for Ruby. It extends the basic Ruby logging facility with log records and parsing abilities.'
-  s_.homepage = 'http://github.com/dazuma/sawmill'
+  s_.homepage = 'http://virtuoso.rubyforge.org/sawmill'
   s_.rubyforge_project = 'virtuoso'
   s_.required_ruby_version = '>= 1.8.6'
   s_.files = ::FileList['lib/**/*.rb', 'tests/**/*.rb', '*.rdoc', 'Rakefile'].to_a
@@ -88,8 +88,8 @@ gemspec_ = ::Gem::Specification.new do |s_|
   s_.has_rdoc = true
   s_.test_files = ::FileList['tests/tc_*.rb']
   s_.platform = ::Gem::Platform::RUBY
-  s_.add_dependency('blockenspiel', '>= 0.2.1')
-  s_.add_dependency('versionomy', '>= 0.1.1')
+  s_.add_dependency('blockenspiel', '>= 0.2.2')
+  s_.add_dependency('versionomy', '>= 0.1.2')
 end
 ::Rake::GemPackageTask.new(gemspec_) do |task_|
   task_.need_zip = false
@@ -106,8 +106,8 @@ task :publish_rdoc_to_rubyforge => [:rerdoc] do
 end
 
 
-# Publish gem
-task :publish_gem_to_rubyforge => [:package] do |t_|
+# Release gem ro rubyforge
+task :release_gem_to_rubyforge => [:package] do |t_|
   v_ = ::ENV["VERSION"]
   abort "Must supply VERSION=x.y.z" unless v_
   if v_ != ::Sawmill::VERSION_STRING
@@ -131,8 +131,8 @@ task :publish_gem_to_rubyforge => [:package] do |t_|
 end
 
 
-# Publish gem
-task :publish_gem_to_gemcutter => [:package] do |t_|
+# Release gem to gemcutter
+task :release_gem_to_gemcutter => [:package] do |t_|
   v_ = ::ENV["VERSION"]
   abort "Must supply VERSION=x.y.z" unless v_
   if v_ != ::Sawmill::VERSION_STRING
@@ -144,4 +144,4 @@ end
 
 
 # Publish everything
-task :publish => [:publish_gem_to_gemcutter, :publish_gem_to_rubyforge, :publish_rdoc_to_rubyforge]
+task :release => [:release_gem_to_gemcutter, :release_gem_to_rubyforge, :publish_rdoc_to_rubyforge]

data/lib/sawmill/entry_processor/filter_basic_fields.rb

@@ -119,7 +119,7 @@ module Sawmill
       def _check_filter(entry_)  # :nodoc:
         if @level
           level_ = entry_.level
-          if @level.kind_of?(::Sawmill::Level)
+          if @level.kind_of?(Level)
             check_level_ = @level
             if level_.group != check_level_.group
               return false unless @accept_incomparable_levels

data/lib/sawmill/entry_processor/format.rb

@@ -74,7 +74,7 @@ module Sawmill
         elsif destination_.respond_to?(:close) && destination_.respond_to?(:write)
           @io = destination_
         else
-          raise ArgumentError, "Unknown destination type"
+          raise ::ArgumentError, "Unknown destination type"
         end
         @include_id = opts_[:include_id]
         @fractional_second_digits = (opts_[:fractional_second_digits] || 2).to_i
@@ -139,7 +139,7 @@ module Sawmill
       
       def unknown_data(entry_)
         return false unless @io || @rotater
-        _write_str(entry_.line+"\n", entry_.record_id)
+        _write_str(entry_.line+"\n", nil)
         true
       end
       

data/lib/sawmill/entry_processor/simple_queue.rb

@@ -70,6 +70,13 @@ module Sawmill
       end
       
       
+      # Return an array of the contents of the queue, in order.
+      
+      def dequeue_all
+        @queue.dequeue_all
+      end
+      
+      
       # Return the size of the queue, which is 0 if the queue is empty.
       
       def size

data/lib/sawmill/entry_processor.rb

@@ -130,12 +130,12 @@ module Sawmill
       
       def _interpret_processor(param_)  # :nodoc:
         case param_
-        when Class
+        when ::Class
           param_.new
         when Base
           param_
         else
-          raise ArgumentError, "Unknown processor object of type #{param_.class.name}"
+          raise ::ArgumentError, "Unknown processor object of type #{param_.class.name}"
         end
       end
       

data/lib/sawmill/log_record_middleware.rb

@@ -62,7 +62,7 @@ module Sawmill
     
     def initialize(app_, logger_=nil, opts_={})
       @app = app_
-      @logger = logger_ || Logger.new(:progname => 'rack', :processor => ::Sawmill::Formatter.new(STDOUT))
+      @logger = logger_ || Logger.new(:progname => 'rack', :processor => Formatter.new(::STDOUT))
       @request_id_key = opts_[:request_id_key] || 'sawmill.request_id'
       @start_time_attribute = opts_[:start_time_attribute]
       @end_time_attribute = opts_[:end_time_attribute]
@@ -72,14 +72,14 @@ module Sawmill
     def call(env_)
       env_[@request_id_key] = @logger.begin_record
       if @start_time_attribute
-        time_ = Time.now.utc
+        time_ = ::Time.now.utc
         @logger.set_attribute(@start_time_attribute, time_.strftime('%Y-%m-%dT%H:%M:%S.') + ('%06d' % time_.usec) + 'Z')
       end
       begin
         return @app.call(env_)
       ensure
         if @end_time_attribute
-          time_ = Time.now.utc
+          time_ = ::Time.now.utc
           @logger.set_attribute(@end_time_attribute, time_.strftime('%Y-%m-%dT%H:%M:%S.') + ('%06d' % time_.usec) + 'Z')
         end
         @logger.end_record

data/lib/sawmill/logger.rb

@@ -34,8 +34,9 @@
 ;
 
 
-if RUBY_VERSION >= '1.9'
+begin
   require 'securerandom'
+rescue ::LoadError
 end
 
 
@@ -80,7 +81,7 @@ module Sawmill
     #   If not specified, log entries are written out to STDOUT.
     
     def initialize(opts_={})
-      @levels = opts_[:levels] || ::Sawmill::STANDARD_LEVELS
+      @levels = opts_[:levels] || STANDARD_LEVELS
       @level = @levels.get(opts_[:level])
       if opts_.include?(:attribute_level)
         @attribute_level = @levels.get(opts_[:attribute_level])
@@ -89,8 +90,8 @@ module Sawmill
       end
       @progname = opts_[:progname] || 'sawmill'
       @record_progname = opts_[:record_progname] || @progname
-      @record_id_generator = opts_[:record_id_generator] || _get_default_record_id_generator
-      @processor = opts_[:processor] || ::Sawmill::Formatter.new(STDOUT)
+      @record_id_generator = opts_[:record_id_generator] || Logger._get_default_record_id_generator
+      @processor = opts_[:processor] || Formatter.new(::STDOUT)
       @current_record_id = nil
     end
     
@@ -122,7 +123,7 @@ module Sawmill
       else
         message_ = message_.inspect
       end
-      @processor.message(Entry::Message.new(level_obj_, Time.now, progname_, @current_record_id, message_))
+      @processor.message(Entry::Message.new(level_obj_, ::Time.now, progname_, @current_record_id, message_))
       true
     end
     alias_method :log, :add
@@ -161,7 +162,7 @@ module Sawmill
     def begin_record(id_=nil)
       end_record if @current_record_id
       @current_record_id = (id_ || @record_id_generator.call).to_s
-      @processor.begin_record(Entry::BeginRecord.new(@levels.highest, Time.now, @record_progname, @current_record_id))
+      @processor.begin_record(Entry::BeginRecord.new(@levels.highest, ::Time.now, @record_progname, @current_record_id))
       @current_record_id
     end
     
@@ -180,7 +181,7 @@ module Sawmill
     
     def end_record
       if @current_record_id
-        @processor.end_record(Entry::EndRecord.new(@levels.highest, Time.now, @record_progname, @current_record_id))
+        @processor.end_record(Entry::EndRecord.new(@levels.highest, ::Time.now, @record_progname, @current_record_id))
         id_ = @current_record_id
         @current_record_id = nil
         id_
@@ -209,7 +210,7 @@ module Sawmill
         end
       end
       return true if level_obj_ < @level
-      @processor.attribute(Entry::Attribute.new(level_obj_, Time.now, progname_ || @record_progname, @current_record_id, key_, value_, operation_))
+      @processor.attribute(Entry::Attribute.new(level_obj_, ::Time.now, progname_ || @record_progname, @current_record_id, key_, value_, operation_))
       true
     end
     
@@ -288,7 +289,7 @@ module Sawmill
     # a default generator which uses the variant 4 (random) UUID standard.
     
     def to_generate_record_id(&block_)
-      @record_id_generator = block_ || _get_default_record_id_generator
+      @record_id_generator = block_ || Logger._get_default_record_id_generator
     end
     
     
@@ -339,21 +340,23 @@ module Sawmill
     end
     
     
-    def _get_default_record_id_generator  # :nodoc:
-      if RUBY_VERSION >= '1.9'
-        lambda do
-          uuid_ = SecureRandom.hex(32)
-          uuid_[12] = '4'
-          uuid_[16] = (uuid_[16,1].to_i(16)&3|8).to_s(16)
-          uuid_.insert(8, '-')
-          uuid_.insert(13, '-')
-          uuid_.insert(18, '-')
-          uuid_.insert(23, '-')
-          uuid_
+    def self._get_default_record_id_generator  # :nodoc:
+      unless @_default_generator
+        if defined?(::SecureRandom)
+          def self._random_hex32
+            ::SecureRandom.hex(32)
+          end
+        elsif defined?(::ActiveSupport::SecureRandom)
+          def self._random_hex32
+            ::ActiveSupport::SecureRandom.hex(32)
+          end
+        else
+          def self._random_hex32
+            ::Kernel.rand(0x100000000000000000000000000000000).to_s(16).rjust(32, '0')
+          end
         end
-      else
-        lambda do
-          uuid_ = Kernel.rand(0x100000000000000000000000000000000).to_s(16).rjust(32, '0')
+        @_default_generator = ::Proc.new do
+          uuid_ = _random_hex32
           uuid_[12] = '4'
           uuid_[16] = (uuid_[16,1].to_i(16)&3|8).to_s(16)
           uuid_.insert(8, '-')
@@ -363,8 +366,8 @@ module Sawmill
           uuid_
         end
       end
+      @_default_generator
     end
-    private :_get_default_record_id_generator
     
     
   end

data/lib/sawmill/multi_parser.rb

@@ -0,0 +1,127 @@
+# -----------------------------------------------------------------------------
+# 
+# Sawmill multi-stream parser utility
+# 
+# -----------------------------------------------------------------------------
+# Copyright 2009 Daniel Azuma
+# 
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of the copyright holder, nor the names of any other
+#   contributors to this software, may be used to endorse or promote products
+#   derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+;
+
+
+module Sawmill
+  
+  
+  # A logfile parser that parses log entries from multiple logfile streams,
+  # sorts by timestamp, and sends them to a processor.
+  
+  class MultiParser
+    
+    
+    # Create a new parser that reads from the given streams.
+    # 
+    # You should provide a processor to receive the data from the logfile.
+    # The processor may be either an entry processor or a record processor.
+    # You may also pass nil for the processor. In this case, the generated
+    # log entries will not be sent to a processor but will still be returned
+    # by the parse_one_entry method.
+    # 
+    # Recognized options include:
+    # 
+    # <tt>:levels</tt>
+    #   Sawmill::LevelGroup to use to parse log levels.
+    #   If not specified, Sawmill::STANDARD_LEVELS is used by default.
+    # <tt>:emit_incomplete_records_at_eof</tt>
+    #   If set to true, causes any incomplete log records to be emitted
+    #   in their incomplete state when EOF is reached on all streams.
+    
+    def initialize(io_array_, processor_, opts_={})
+      @emit_incomplete_records_at_eof = opts_.delete(:emit_incomplete_records_at_eof)
+      @heap = Util::Heap.new{ |a_, b_| a_[1].timestamp <=> b_[1].timestamp }
+      @queue = Util::Queue.new
+      io_array_.each{ |io_| _enqueue(Parser.new(io_, nil, opts_)) }
+      @processor = nil
+      if processor_.respond_to?(:record) && processor_.respond_to?(:extra_entry)
+        @processor = RecordBuilder.new(processor_)
+      elsif processor_.respond_to?(:begin_record) && processor_.respond_to?(:end_record)
+        @processor = processor_
+      end
+      @classifier = @processor ? EntryClassifier.new(@processor) : nil
+    end
+    
+    
+    # Parse one log entry from the streams and emit it to the processor.
+    # Also returns the log entry.
+    # Returns nil if EOF has been reached on all streams.
+    
+    def parse_one_entry
+      entry_ = @queue.dequeue
+      unless entry_
+        data_ = @heap.remove
+        if data_
+          _enqueue(data_[0])
+          entry_ = data_[1]
+        else
+          if @emit_incomplete_records_at_eof && @processor.respond_to?(:emit_incomplete_records)
+            @processor.emit_incomplete_records
+          end
+        end
+      end
+      @classifier.entry(entry_) if entry_
+      entry_
+    end
+    
+    
+    # Parse until EOF is reached on all streams, and emit the log
+    # entries to the processor.
+    
+    def parse_all
+      while parse_one_entry; end
+    end
+    
+    
+    private
+    
+    def _enqueue(parser_)  # :nodoc:
+      loop do
+        entry_ = parser_.parse_one_entry
+        return unless entry_
+        if entry_.type == :unknown_data
+          @queue.enqueue(entry_)
+        else
+          @heap << [parser_, entry_]
+          return
+        end
+      end
+    end
+    
+    
+  end
+  
+  
+end

data/lib/sawmill/parser.rb

@@ -62,21 +62,20 @@ module Sawmill
     # <tt>:levels</tt>
     #   Sawmill::LevelGroup to use to parse log levels.
     #   If not specified, Sawmill::STANDARD_LEVELS is used by default.
-    # <tt>:emit_incomplete_records_on_complete</tt>
+    # <tt>:emit_incomplete_records_at_eof</tt>
     #   If set to true, causes any incomplete log records to be emitted
     #   in their incomplete state when EOF is reached.
     
     def initialize(io_, processor_, opts_={})
       @io = io_
-      @record_processor = nil
       @processor = nil
       if processor_.respond_to?(:record) && processor_.respond_to?(:extra_entry)
         @processor = RecordBuilder.new(processor_)
       elsif processor_.respond_to?(:begin_record) && processor_.respond_to?(:end_record)
         @processor = processor_
       end
-      @levels = opts_[:levels] || ::Sawmill::STANDARD_LEVELS
-      @emit_incomplete_records_on_complete = opts_[:emit_incomplete_records_on_complete]
+      @levels = opts_[:levels] || STANDARD_LEVELS
+      @emit_incomplete_records_at_eof = opts_[:emit_incomplete_records_at_eof]
       @current_record_id = nil
       @parser_directives = {}
     end
@@ -155,11 +154,11 @@ module Sawmill
           if str_ =~ DIRECTIVE_REGEXP
             @parser_directives[$1] = $2
           end
-          entry_ = Entry::UnknownData.new(str_)
+          entry_ = Entry::UnknownData.new(str_.chomp)
           @processor.unknown_data(entry_) if @processor.respond_to?(:unknown_data)
         end
       else
-        if @emit_incomplete_records_on_complete && @processor.respond_to?(:emit_incomplete_records)
+        if @emit_incomplete_records_at_eof && @processor.respond_to?(:emit_incomplete_records)
           @processor.emit_incomplete_records
         end
       end
@@ -171,7 +170,7 @@ module Sawmill
     # entries to the processor.
     
     def parse_all
-      while process_one_entry; end
+      while parse_one_entry; end
     end
     
     

data/lib/sawmill/record.rb

@@ -89,6 +89,16 @@ module Sawmill
     end
     
     
+    def eql?(obj_)  # :nodoc:
+      return false unless obj_.kind_of?(Record)
+      return @entries == obj_.instance_variable_get(:@entries)
+    end
+    
+    def ==(obj_)  # :nodoc:
+      eql?(obj_)
+    end
+    
+    
     # Append a log entry to this record.
     # 
     # Entries must be added in order. Raises Errors::IllegalRecordError if
@@ -132,9 +142,9 @@ module Sawmill
         when :append
           val_ = @attributes[entry_.key]
           case val_
-          when Array
+          when ::Array
             val_ << entry_.value
-          when String
+          when ::String
             @attributes[entry_.key] = [val_, entry_.value]
           when nil
             @attributes[entry_.key] = [entry_.value]

data/lib/sawmill/record_processor/decompose.rb

@@ -49,7 +49,7 @@ module Sawmill
       
       def initialize(processor_, opts_={})
         @processor = processor_
-        @classifier = ::Sawmill::EntryClassifier.new(processor_)
+        @classifier = EntryClassifier.new(processor_)
       end
       
       

data/lib/sawmill/record_processor/filter_by_attributes.rb

@@ -61,9 +61,9 @@ module Sawmill
         @attributes.each do |key_, value_|
           record_value_ = record_.attribute(key_.to_s)
           case record_value_
-          when Array
+          when ::Array
             return false unless record_value_.find{ |rval_| value_ === rval_ }
-          when String
+          when ::String
             return false unless value_ === record_value_
           when nil
             return false unless value_.nil?

data/lib/sawmill/record_processor/simple_queue.rb

@@ -73,6 +73,13 @@ module Sawmill
       end
       
       
+      # Return an array of the contents of the record queue, in order.
+      
+      def dequeue_all
+        @queue.dequeue_all
+      end
+      
+      
       # Return the number of records in the record queue.
       
       def size
@@ -88,6 +95,13 @@ module Sawmill
       end
       
       
+      # Return an array of the contents of the extra entry queue, in order.
+      
+      def dequeue_all_extra_entries
+        @extra_entries_queue.dequeue_all
+      end
+      
+      
       # Return the number of entries in the extra entry queue.
       
       def extra_entries_size

data/lib/sawmill/record_processor.rb

@@ -109,12 +109,12 @@ module Sawmill
       
       def _interpret_processor(param_)  # :nodoc:
         case param_
-        when Class
+        when ::Class
           param_.new
         when Base
           param_
         else
-          raise ArgumentError, "Unknown processor object of type #{param_.class.name}"
+          raise ::ArgumentError, "Unknown processor object of type #{param_.class.name}"
         end
       end
       

data/lib/sawmill/rotater/base.rb

@@ -63,7 +63,7 @@ module Sawmill
       # closed in the meantime.
       
       def open_handle(handle_)
-        STDOUT
+        ::STDOUT
       end
       
       

data/lib/sawmill/rotater/date_based_log_file.rb

@@ -82,8 +82,8 @@ module Sawmill
       
       def initialize(options_)
         @turnover_frequency = options_[:turnover_frequency] || :none
-        dirname_ = options_[:dirname] || Dir.getwd
-        @prefix = File.join(dirname_, options_[:prefix] || 'sawmill')
+        dirname_ = options_[:dirname] || ::Dir.getwd
+        @prefix = ::File.join(dirname_, options_[:prefix] || 'sawmill')
         @suffix = options_[:suffix] || '.log'
         @local_timezone = options_[:local_timezone]
         @date_pattern =
@@ -101,7 +101,7 @@ module Sawmill
       
       def preferred_handle
         if @date_pattern
-          time_ = Time.now
+          time_ = ::Time.now
           time_.utc unless @local_timezone
           time_.strftime(@date_pattern)
         else
@@ -118,7 +118,7 @@ module Sawmill
         else
           path_ = @prefix+@suffix
         end
-        file_ = File.open(path_, File::CREAT | File::WRONLY | File::APPEND)
+        file_ = ::File.open(path_, ::File::CREAT | ::File::WRONLY | ::File::APPEND)
         file_.sync = true
         file_
       end

data/lib/sawmill/rotater/shifting_log_file.rb

@@ -91,11 +91,11 @@ module Sawmill
         end
         @history_size = options_[:history_size].to_i
         @history_size = 1 if @history_size < 1 && (@max_logfile_size || @shift_period)
-        dirname_ = options_[:dirname] || Dir.getwd
-        @normal_path = File.join(dirname_, options_[:filename] || 'sawmill.log')
+        dirname_ = options_[:dirname] || ::Dir.getwd
+        @normal_path = ::File.join(dirname_, options_[:filename] || 'sawmill.log')
         @preferred_handle = 0
         @open_handles = {}
-        @last_shift = Time.now
+        @last_shift = ::Time.now
       end
       
       
@@ -114,7 +114,7 @@ module Sawmill
         else
           path_ = "#{@normal_path}.#{@preferred_handle-handle_-1}"
         end
-        file_ = File.open(path_, File::CREAT | File::WRONLY | File::APPEND)
+        file_ = ::File.open(path_, ::File::CREAT | ::File::WRONLY | ::File::APPEND)
         file_.sync = true
         @open_handles[handle_] = true
         file_
@@ -126,7 +126,7 @@ module Sawmill
       def close_handle(handle_, io_)
         io_.close
         if @preferred_handle - handle_ > @history_size
-          File.delete("#{@normal_path}.#{@preferred_handle-handle_-1}") rescue nil
+          ::File.delete("#{@normal_path}.#{@preferred_handle-handle_-1}") rescue nil
         end
         @open_handles.delete(handle_)
         nil
@@ -138,22 +138,22 @@ module Sawmill
       def before_write
         return unless @max_logfile_size || @shift_period
         turnover_ = false
-        if @max_logfile_size && File.file?(@normal_path) && File.size(@normal_path) > @max_logfile_size
+        if @max_logfile_size && ::File.file?(@normal_path) && ::File.size(@normal_path) > @max_logfile_size
           turnover_ = true
         end
-        if @shift_period && (Time.now - @last_shift) > @shift_period
+        if @shift_period && (::Time.now - @last_shift) > @shift_period
           turnover_ = true
         end
         if turnover_
           max_ = @preferred_handle - @open_handles.keys.min + 1
           max_ = @history_size if max_ < @history_size
-          File.delete("#{@normal_path}.#{max_-1}") rescue nil
+          ::File.delete("#{@normal_path}.#{max_-1}") rescue nil
           (max_-1).downto(1) do |index_|
-            File.rename("#{@normal_path}.#{index_-1}", "#{@normal_path}.#{index_}") rescue nil
+            ::File.rename("#{@normal_path}.#{index_-1}", "#{@normal_path}.#{index_}") rescue nil
           end
-          File.rename("#{@normal_path}", "#{@normal_path}.0") rescue nil
+          ::File.rename("#{@normal_path}", "#{@normal_path}.0") rescue nil
           @preferred_handle += 1
-          @last_shift = Time.now
+          @last_shift = ::Time.now
         end
       end
       

data/lib/sawmill/rotater.rb

@@ -73,13 +73,13 @@ module Sawmill
     
     def initialize(io_manager_, opts_={})
       @omit_directives = opts_.delete(:omit_directives)
-      if io_manager_.kind_of?(Class)
+      if io_manager_.kind_of?(::Class)
         @io_manager = io_manager_.new(opts_)
       else
         @io_manager = io_manager_
       end
       @handles ||= {}
-      @mutex ||= Monitor.new
+      @mutex ||= ::Monitor.new
     end
     
     

data/lib/sawmill/util/heap.rb

@@ -0,0 +1,153 @@
+# -----------------------------------------------------------------------------
+# 
+# Sawmill heap utility
+# 
+# -----------------------------------------------------------------------------
+# Copyright 2009 Daniel Azuma
+# 
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of the copyright holder, nor the names of any other
+#   contributors to this software, may be used to endorse or promote products
+#   derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+;
+
+
+module Sawmill
+  
+  module Util
+    
+    
+    # A simple heap class.
+    
+    class Heap
+      
+      
+      # Create a new heap.
+      
+      def initialize(data_=nil, &block_)
+        @_heap = data_ || []
+        @_comparator = block_ || ::Proc.new{ |a_,b_| a_ <=> b_ }
+      end
+      
+      
+      def merge(enum_)
+        enum_.each{ |value_| add(value_) }
+        self
+      end
+      
+      
+      def add(value_)
+        @_heap << value_
+        _sift_up(@_heap.length-1)
+        self
+      end
+      
+      
+      def <<(value_)
+        add(value_)
+      end
+      
+      
+      def remove
+        ret_ = @_heap[0]
+        if @_heap.length > 1
+          @_heap[0] = @_heap.pop
+          _sift_down(0)
+        else
+          @_heap.clear
+        end
+        ret_
+      end
+      
+      
+      def peek
+        @_heap[0]
+      end
+      
+      
+      def size
+        @_heap.size
+      end
+      
+      
+      def empty?
+        @_heap.empty?
+      end
+      
+      
+      def clear
+        @_heap.clear
+      end
+      
+      
+      def each!
+        while !empty?
+          yield(remove)
+        end
+      end
+      
+      
+      private
+      
+      def _sift_up(start_)  # :nodoc:
+        while start_ > 0
+          parent_ = (start_ + 1) / 2 - 1
+          if @_comparator.call(@_heap[start_], @_heap[parent_]) < 0
+            @_heap[start_], @_heap[parent_] = @_heap[parent_], @_heap[start_]
+            start_ = parent_
+          else
+            return start_
+          end
+        end
+      end
+      
+      
+      def _sift_down(start_)  # :nodoc:
+        length_ = self.size
+        while length_ >= (child2_ = (start_ + 1) * 2)
+          child1_ = child2_-1
+          if length_ <= child2_
+            earliest_child_ = child1_
+          elsif @_comparator.call(@_heap[child1_], @_heap[child2_]) < 0
+            earliest_child_ = child1_
+          else
+            earliest_child_ = child2_
+          end
+          if @_comparator.call(@_heap[start_], @_heap[earliest_child_]) < 0
+            return start_
+          else
+            @_heap[start_], @_heap[earliest_child_] = @_heap[earliest_child_], @_heap[start_]
+            start_ = earliest_child_
+          end
+        end
+      end
+      
+      
+    end
+    
+    
+  end
+  
+end

data/lib/sawmill/util/queue.rb

@@ -1,6 +1,6 @@
 # -----------------------------------------------------------------------------
 # 
-# Sawmill entry stream processor interface
+# Sawmill queue utility
 # 
 # -----------------------------------------------------------------------------
 # Copyright 2009 Daniel Azuma
@@ -101,6 +101,7 @@ module Sawmill
         if @push_ptr
           if @pop_ptr
             object_ = @buffer[@pop_ptr]
+            @buffer[@pop_ptr] = nil
             @pop_ptr += 1
             @pop_ptr = 0 if @pop_ptr == @buffer.size
             @pop_ptr = nil if @pop_ptr == @push_ptr
@@ -114,6 +115,31 @@ module Sawmill
       end
       
       
+      # Return an array of the contents of the queue, in order.
+      
+      def dequeue_all
+        if @push_ptr
+          if @pop_ptr
+            if @pop_ptr < @push_ptr
+              ret_ = @buffer[@pop_ptr..@push_ptr-1]
+            else
+              ret_ = @buffer[@pop_ptr..-1] + @buffer[0..@push_ptr-1]
+            end
+            @buffer.fill(nil)
+            @push_ptr = 0
+            @pop_ptr = nil
+            ret_
+          else
+            []
+          end
+        else
+          ret_ = @buffer
+          @buffer = []
+          ret_
+        end
+      end
+      
+      
       # Return the size of the queue, which is 0 if the queue is empty.
       
       def size

data/lib/sawmill/version.rb

@@ -39,7 +39,7 @@
 module Sawmill
   
   # Current gem version, as a frozen string.
-  VERSION_STRING = '0.0.1'.freeze
+  VERSION_STRING = '0.0.2'.freeze
   
   # Current gem version, as a Versionomy::Value.
   VERSION = ::Versionomy.parse(VERSION_STRING, :standard)

data/lib/sawmill.rb

@@ -49,6 +49,7 @@ dir_ = ::File.expand_path('sawmill', ::File.dirname(__FILE__))
 includes_ = [
  'version',
  'util/queue',
+ 'util/heap',
  'errors',
  'level',
  'entry',
@@ -68,6 +69,7 @@ includes_ = [
  'record_processor/decompose',
  'record_processor/format',
  'parser',
+ 'multi_parser',
  'logger',
  'rotater',
  'rotater/base',

data/tests/tc_multi_parser.rb

@@ -0,0 +1,88 @@
+# -----------------------------------------------------------------------------
+# 
+# Sawmill: tests multi-parser
+# 
+# -----------------------------------------------------------------------------
+# Copyright 2009 Daniel Azuma
+# 
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of the copyright holder, nor the names of any other
+#   contributors to this software, may be used to endorse or promote products
+#   derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+# -----------------------------------------------------------------------------
+
+
+require 'test/unit'
+require 'stringio'
+require ::File.expand_path("#{::File.dirname(__FILE__)}/../lib/sawmill.rb")
+
+
+module Sawmill
+  module Tests  # :nodoc:
+    
+    class TestMultiParser < ::Test::Unit::TestCase  # :nodoc:
+      
+      
+      def setup
+        @levels = ::Sawmill::STANDARD_LEVELS
+      end
+      
+      
+      def _get_io_array(entry_groups_)
+        strings_ = []
+        entry_groups_.each do |entries_|
+          stringio_ = ::StringIO.new
+          formatter_ = ::Sawmill::EntryClassifier.new(::Sawmill::Formatter.new(stringio_, :fractional_second_digits => 6))
+          entries_.each do |entry_|
+            formatter_.entry(entry_)
+          end
+          strings_ << stringio_.string
+        end
+        strings_.map{ |str_| ::StringIO.new(str_) }
+      end
+      
+      
+      # Test interleaved entries including unknown data entries.
+      # Makes sure they come out in the right order.
+      
+      def test_interleaved_entries
+        base_time_ = Time.now.utc
+        entries_ = []
+        2.times do |i_|
+          entries_ << ::Sawmill::Entry::UnknownData.new("Unknown #{i_}")
+        end
+        4.times do |i_|
+          entries_ << ::Sawmill::Entry::Message.new(@levels.get(:INFO), base_time_+i_, 'rails', nil, "Hello #{i_}")
+        end
+        io_array_ = _get_io_array([[entries_[0], entries_[2], entries_[5]], [entries_[3], entries_[4], entries_[1]]])
+        queue_ = ::Sawmill::EntryProcessor::SimpleQueue.new
+        ::Sawmill::MultiParser.new(io_array_, queue_).parse_all
+        assert_equal([entries_[0], entries_[2], entries_[3], entries_[4], entries_[1], entries_[5]], queue_.dequeue_all)
+      end
+      
+      
+    end
+    
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: sawmill
 version: !ruby/object:Gem::Version 
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors: 
 - Daniel Azuma
@@ -20,7 +20,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.2.1
+        version: 0.2.2
     version: 
 - !ruby/object:Gem::Dependency 
   name: versionomy
@@ -30,7 +30,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.1.1
+        version: 0.1.2
     version: 
 description: Sawmill is a logging and log analysis system for Ruby. It extends the basic Ruby logging facility with log records and parsing abilities.
 email: dazuma@gmail.com
@@ -54,6 +54,7 @@ files:
 - lib/sawmill/level.rb
 - lib/sawmill/log_record_middleware.rb
 - lib/sawmill/logger.rb
+- lib/sawmill/multi_parser.rb
 - lib/sawmill/parser.rb
 - lib/sawmill/record.rb
 - lib/sawmill/record_processor/conditionals.rb
@@ -67,6 +68,7 @@ files:
 - lib/sawmill/rotater/date_based_log_file.rb
 - lib/sawmill/rotater/shifting_log_file.rb
 - lib/sawmill/rotater.rb
+- lib/sawmill/util/heap.rb
 - lib/sawmill/util/queue.rb
 - lib/sawmill/version.rb
 - lib/sawmill.rb
@@ -74,13 +76,14 @@ files:
 - tests/tc_formatter_parser.rb
 - tests/tc_levels.rb
 - tests/tc_logger.rb
+- tests/tc_multi_parser.rb
 - tests/tc_record_processors.rb
 - tests/tc_records.rb
 - History.rdoc
 - README.rdoc
 - Rakefile
 has_rdoc: true
-homepage: http://github.com/dazuma/sawmill
+homepage: http://virtuoso.rubyforge.org/sawmill
 licenses: []
 
 post_install_message: 
@@ -112,5 +115,6 @@ test_files:
 - tests/tc_formatter_parser.rb
 - tests/tc_levels.rb
 - tests/tc_logger.rb
+- tests/tc_multi_parser.rb
 - tests/tc_record_processors.rb
 - tests/tc_records.rb