sawmill 0.0.4 → 0.0.5

data/History.rdoc

@@ -1,4 +1,11 @@
-=== 0.0.4 / 2009-??-??
+=== 0.0.5 / 2009-11-08
+
+* API CHANGE: Reworked how string encoding is specified. Default encoding for both reading and writing is now ASCII-8BIT unless specified in the API or within the file as a parser directive.
+* Handle encoding errors more gracefully by replacing characters rather than raising exceptions.
+* Added a parser directive to set logfile encoding.
+* Versionomy is no longer a hard dependency-- it is now used only if available.
+
+=== 0.0.4 / 2009-11-06
 
 * API CHANGE: Renamed DateBasedLogFile options :prefix and :suffix to :path_prefix and :path_suffix
 * API CHANGE: Renamed ShiftingLogFile options :filepath and :max_logfile_size to :file_path and :max_file_size

data/README.rdoc

@@ -26,7 +26,7 @@ Sawmill is an extension to the standard ruby Logger mechanism that enables autom
 === Requirements
 
 * Ruby 1.8.6 or later (1.8.7 recommended), Ruby 1.9.1 or later, or JRuby 1.4 or later.
-* versionomy gem.
+* Blockenspiel 0.3.1 or later.
 
 === Installation
 

data/Rakefile

@@ -88,8 +88,7 @@ gemspec_ = ::Gem::Specification.new do |s_|
   s_.has_rdoc = true
   s_.test_files = ::FileList['tests/tc_*.rb']
   s_.platform = ::Gem::Platform::RUBY
-  s_.add_dependency('blockenspiel', '>= 0.3.0')
-  s_.add_dependency('versionomy', '>= 0.2.0')
+  s_.add_dependency('blockenspiel', '>= 0.3.1')
 end
 ::Rake::GemPackageTask.new(gemspec_) do |task_|
   task_.need_zip = false

data/lib/sawmill.rb

@@ -34,14 +34,7 @@
 ;
 
 
-begin
-  require 'blockenspiel'
-  require 'versionomy'
-rescue ::LoadError
-  require 'rubygems'
-  require 'blockenspiel'
-  require 'versionomy'
-end
+require 'blockenspiel'
 
 
 dir_ = ::File.expand_path('sawmill', ::File.dirname(__FILE__))

data/lib/sawmill/interface.rb

@@ -266,11 +266,13 @@ module Sawmill
     # <tt>:encoding</tt>::
     #   Specify an encoding for file data. (Ruby 1.9 only.)
     #   You may specify an encoding name or an encoding object.
-    #   If not specified, uses the default external encoding.
+    #   If not specified, reads raw bytes (e.g. defaults to 'ASCII-8BIT').
+    #   Note that the encoding may also be modified by the file itself,
+    #   if an appropriate parser directive is encountered.
     # <tt>:internal_encoding</tt>::
     #   Specify an encoding to transcode to. (Ruby 1.9 only.)
     #   You may specify an encoding name or an encoding object.
-    #   If not specified, uses the default internal encoding.
+    #   If not specified, uses the encoding as read from the file.
     
     def open_entries(globs_, opts_={}, &block_)
       processor_ = EntryProcessor.build(&block_)
@@ -297,11 +299,13 @@ module Sawmill
     # <tt>:encoding</tt>::
     #   Specify an encoding for file data. (Ruby 1.9 only.)
     #   You may specify an encoding name or an encoding object.
-    #   If not specified, uses the default external encoding.
+    #   If not specified, reads raw bytes (e.g. defaults to 'ASCII-8BIT').
+    #   Note that the encoding may also be modified by the file itself,
+    #   if an appropriate parser directive is encountered.
     # <tt>:internal_encoding</tt>::
     #   Specify an encoding to transcode to. (Ruby 1.9 only.)
     #   You may specify an encoding name or an encoding object.
-    #   If not specified, uses the default internal encoding.
+    #   If not specified, uses the encoding as read from the file.
     
     def open_records(globs_, opts_={}, &block_)
       processor_ = RecordProcessor.build(&block_)
@@ -331,11 +335,13 @@ module Sawmill
     # <tt>:encoding</tt>::
     #   Specify an encoding for file data. (Ruby 1.9 only.)
     #   You may specify an encoding name or an encoding object.
-    #   If not specified, uses the default external encoding.
+    #   If not specified, reads raw bytes (e.g. defaults to 'ASCII-8BIT').
+    #   Note that the encoding may also be modified by the file itself,
+    #   if an appropriate parser directive is encountered.
     # <tt>:internal_encoding</tt>::
     #   Specify an encoding to transcode to. (Ruby 1.9 only.)
     #   You may specify an encoding name or an encoding object.
-    #   If not specified, uses the default internal encoding.
+    #   If not specified, uses the encoding as read from the file.
     
     def open_files(globs_, processor_, opts_={})
       finish_opt_ = opts_.delete(:finish)
@@ -343,20 +349,17 @@ module Sawmill
       internal_encoding_ = opts_.delete(:internal_encoding)
       mode_ = 'r'
       if defined?(::Encoding)
-        if encoding_
-          encoding_ = ::Encoding.find(encoding_) unless encoding_.respond_to?(:name)
-          encoding_ = nil if encoding_ == ::Encoding.default_external
+        if !encoding_
+          encoding_ = ::Encoding::ASCII_8BIT
+        elsif encoding_ && !encoding_.respond_to?(:name)
+          encoding_ = ::Encoding.find(encoding_)
         end
-        if internal_encoding_
-          internal_encoding_ = ::Encoding.find(internal_encoding_) unless internal_encoding_.respond_to?(:name)
-          internal_encoding_ = nil if internal_encoding_ == ::Encoding.default_internal
+        if internal_encoding_ && !internal_encoding_.respond_to?(:name)
+          internal_encoding_ = ::Encoding.find(internal_encoding_)
         end
         if encoding_
           mode_ << ":#{encoding_.name}"
-        elsif internal_encoding_
-          mode_ << ":#{::Encoding.default_external.name}"
         end
-        mode_ << ":#{internal_encoding_.name}" if internal_encoding_
       else
         encoding_ = nil
         internal_encoding_ = nil
@@ -376,7 +379,7 @@ module Sawmill
             else
               io_array_ << ::File.open(path_, mode_)
               encoding_array_ << nil
-              internal_encoding_array_ << nil
+              internal_encoding_array_ << internal_encoding_
             end
           end
         end

data/lib/sawmill/parser.rb

@@ -46,6 +46,8 @@ module Sawmill
     LINE_REGEXP = /^\[\s*([[:graph:]]+)\s+(\d{4})-(\d{2})-(\d{2})(T|\s)(\d{2}):(\d{2}):(\d{2})(.(\d{1,6}))?Z?\s?([+-]\d{4})?\s+([[:graph:]]+)(\s+([[:graph:]]+))?\s+([\^$.=])\]\s(.*)$/
     DIRECTIVE_REGEXP = /^#\s+sawmill_format:\s+(\w+)=(.*)$/
     ATTRIBUTE_REGEXP = /^([[:graph:]]+)\s([=+\/-])\s/
+    SUPPORTS_ENCODING = defined?(::Encoding)
+    ENCODING_OPTS = {:invalid => :replace, :undef => :replace}
     # :startdoc:
     
     
@@ -65,6 +67,16 @@ module Sawmill
     # <tt>:emit_incomplete_records_at_eof</tt>
     #   If set to true, causes any incomplete log records to be emitted
     #   in their incomplete state when EOF is reached.
+    # <tt>:encoding</tt>
+    #   Overrides the IO encoding. (Ruby 1.9 only). If specified, lines
+    #   read from the stream are assumed to be in this encoding. If not
+    #   specified, the IO's default encoding is honored.
+    #   Note that the encoding may also be modified by the stream itself,
+    #   if an appropriate parser directive is encountered.
+    # <tt>:internal_encoding</tt>
+    #   Transcodes strings as they are read. (Ruby 1.9 only). If specified,
+    #   lines are transcoded into this encoding after they are read from
+    #   the stream. If not specified, no post-transcoding is done.
     
     def initialize(io_, processor_, opts_={})
       @io = io_
@@ -77,11 +89,10 @@ module Sawmill
       @levels = opts_[:levels] || STANDARD_LEVELS
       @emit_incomplete_records_at_eof = opts_[:emit_incomplete_records_at_eof]
       @current_record_id = nil
-      @parser_directives = {}
-      @encoding = opts_[:encoding]
-      @internal_encoding = opts_[:internal_encoding]
-      if defined?(::Encoding)
+      if SUPPORTS_ENCODING
+        @encoding = opts_[:encoding]
         @encoding = ::Encoding.find(@encoding) if @encoding && !@encoding.kind_of?(::Encoding)
+        @internal_encoding = opts_[:internal_encoding]
         @internal_encoding = ::Encoding.find(@internal_encoding) if @internal_encoding && !@internal_encoding.kind_of?(::Encoding)
       end
     end
@@ -158,7 +169,7 @@ module Sawmill
           end
         else
           if str_ =~ DIRECTIVE_REGEXP
-            @parser_directives[$1] = $2
+            _set_parser_directive($1, $2)
           end
           entry_ = Entry::UnknownData.new(str_.chomp)
           @processor.unknown_data(entry_) if @processor.respond_to?(:unknown_data)
@@ -185,14 +196,25 @@ module Sawmill
     
     def _get_next_line  # :nodoc:
       str_ = @io.gets
-      if str_
+      if str_ && SUPPORTS_ENCODING
         str_.force_encoding(@encoding) if @encoding
-        str_.encode!(@internal_encoding) if @internal_encoding
+        str_.encode!(@internal_encoding, ENCODING_OPTS) if @internal_encoding
       end
       str_
     end
     
     
+    def _set_parser_directive(key_, value_)  # :nodoc:
+      case key_
+      when 'encoding'
+        if SUPPORTS_ENCODING
+          encoding_ = ::Encoding.find(value_) rescue nil
+          @encoding = encoding_ if encoding_
+        end
+      end
+    end
+    
+    
   end
   
   

data/lib/sawmill/rotater.rb

@@ -59,6 +59,11 @@ module Sawmill
   
   class Rotater
     
+    # :stopdoc:
+    SUPPORTS_ENCODING = defined?(::Encoding)
+    ENCODING_OPTS = {:invalid => :replace, :undef => :replace}
+    # :startdoc:
+    
     
     # Create a rotater using the given rotation strategy.
     # See Sawmill::Rotater::DateBasedLogFile and
@@ -70,9 +75,26 @@ module Sawmill
     # 
     # <tt>:omit_directives</tt>::
     #   If true, omit standard logfile directives. Default is false.
+    # <tt>:concurrent_writes</tt>::
+    #   Set this to true if you expect multiple processes to attempt to
+    #   write to the same log file simultaneously. This option causes the
+    #   rotater to surround writes with an acquisition of the cooperative
+    #   filesystem lock (if available) for the logfile, in an attempt to
+    #   prevent lines from interleaving in one another. Default is false.
+    # <tt>:encoding</tt>::
+    #   Specify an encoding for file data. (Ruby 1.9 only).
+    #   You may pass either an encoding object or an encoding name.
+    #   If not specified, writes raw bytes (e.g. defaults to ASCII-8BIT).
     
     def initialize(io_manager_, opts_={})
       @omit_directives = opts_.delete(:omit_directives)
+      @concurrent_writes = opts_.delete(:concurrent_writes)
+      if SUPPORTS_ENCODING
+        @encoding = opts_.delete(:encoding)
+        if @encoding && !@encoding.respond_to?(:name)
+          @encoding = ::Encoding.find(@encoding)
+        end
+      end
       if io_manager_.kind_of?(::Class)
         @io_manager = io_manager_.new(opts_)
       else
@@ -99,6 +121,23 @@ module Sawmill
     end
     
     
+    def _write_to_stream(io_, str_)
+      if SUPPORTS_ENCODING && @encoding
+        str_ = str_.encode(@encoding, ENCODING_OPTS)
+      end
+      if @concurrent_writes
+        begin
+          io_.flock(::File::LOCK_EX)
+          io_.write(str_)
+        ensure
+          io_.flock(::File::LOCK_UN)
+        end
+      else
+        io_.write(str_)
+      end
+    end
+    
+    
     def _obtain_handle  # :nodoc:
       handle_ = @io_manager.preferred_handle
       if @handles.include?(handle_)
@@ -106,7 +145,13 @@ module Sawmill
       else
         io_ = @io_manager.open_handle(handle_)
         unless @omit_directives
-          io_.write("# sawmill_format: version=1\n")
+          _write_to_stream(io_, "# sawmill_format: version=1\n")
+          if defined?(::Encoding)
+            encoding_ = io_.encoding
+            if encoding_
+              _write_to_stream(io_, "# sawmill_format: encoding=#{encoding_.name}\n")
+            end
+          end
         end
         @handles[handle_] = [handle_, io_, 1]
       end
@@ -149,7 +194,7 @@ module Sawmill
           handle_ = _check_rotate_handle(handle_)
         end
         info_ = @handles[handle_]
-        info_[1].write(str_)
+        _write_to_stream(info_[1], str_)
         handle_
       end
     end

data/lib/sawmill/rotater/date_based_log_file.rb

@@ -79,9 +79,6 @@ module Sawmill
       # <tt>:local_datestamps</tt>::
       #   If true, use the local timezone to create datestamps.
       #   The default is to use UTC.
-      # <tt>:encoding</tt>::
-      #   Specify an encoding name for file data. (Ruby 1.9 only)
-      #   If not specified, uses the default external encoding.
       
       def initialize(options_)
         @turnover_frequency = options_[:turnover_frequency] || :none
@@ -97,10 +94,6 @@ module Sawmill
           when :hourly then "%Y-%m-%d-%H"
           else nil
           end
-        @mode = 'a'
-        if defined?(::Encoding) && (encoding_ = options_[:encoding])
-          @mode << ":#{encoding_}"
-        end
       end
       
       
@@ -125,7 +118,7 @@ module Sawmill
         else
           path_ = @prefix+@suffix
         end
-        file_ = ::File.open(path_, @mode)
+        file_ = ::File.open(path_, 'a')
         file_.sync = true
         file_
       end

data/lib/sawmill/rotater/shifting_log_file.rb

@@ -76,9 +76,6 @@ module Sawmill
       #   The maximum number of old logfiles (files with indexes) to
       #   keep. Files beyond this history size will be automatically
       #   deleted. Default is 1. This value must be at least 1.
-      # <tt>:encoding</tt>::
-      #   Specify an encoding name for file data. (Ruby 1.9 only)
-      #   If not specified, uses the default external encoding.
       
       def initialize(options_)
         @max_logfile_size = options_[:max_file_size] || options_[:max_logfile_size]
@@ -100,10 +97,6 @@ module Sawmill
         @preferred_handle = 0
         @open_handles = {}
         @last_shift = ::Time.now
-        @mode = 'a'
-        if defined?(::Encoding) && (encoding_ = options_[:encoding])
-          @mode << ":#{encoding_}"
-        end
       end
       
       
@@ -122,7 +115,7 @@ module Sawmill
         else
           path_ = "#{@normal_path}.#{@preferred_handle-handle_-1}"
         end
-        file_ = ::File.open(path_, @mode)
+        file_ = ::File.open(path_, 'a')
         file_.sync = true
         @open_handles[handle_] = true
         file_

data/lib/sawmill/version.rb

@@ -34,12 +34,19 @@
 ;
 
 
+begin
+  require 'versionomy'
+rescue ::LoadError
+end
+
+
 module Sawmill
   
   # Current gem version, as a frozen string.
-  VERSION_STRING = '0.0.4'.freeze
+  VERSION_STRING = '0.0.5'.freeze
   
-  # Current gem version, as a Versionomy::Value.
-  VERSION = ::Versionomy.parse(VERSION_STRING, :standard)
+  # Current gem version, as a Versionomy::Value if the versionomy library
+  # is available, or as a frozen string if not.
+  VERSION = defined?(::Versionomy) ? ::Versionomy.parse(VERSION_STRING, :standard) : VERSION_STRING
   
 end

data/tests/tc_entry_processors.rb

@@ -33,6 +33,7 @@
 # -----------------------------------------------------------------------------
 
 
+require 'rubygems'
 require 'test/unit'
 require ::File.expand_path("#{::File.dirname(__FILE__)}/../lib/sawmill.rb")
 

data/tests/tc_formatter_parser.rb

@@ -33,6 +33,7 @@
 # -----------------------------------------------------------------------------
 
 
+require 'rubygems'
 require 'test/unit'
 require 'stringio'
 require ::File.expand_path("#{::File.dirname(__FILE__)}/../lib/sawmill.rb")

data/tests/tc_levels.rb

@@ -33,6 +33,7 @@
 # -----------------------------------------------------------------------------
 
 
+require 'rubygems'
 require 'test/unit'
 require 'logger'
 require ::File.expand_path("#{::File.dirname(__FILE__)}/../lib/sawmill.rb")

data/tests/tc_logger.rb

@@ -33,6 +33,7 @@
 # -----------------------------------------------------------------------------
 
 
+require 'rubygems'
 require 'test/unit'
 require ::File.expand_path("#{::File.dirname(__FILE__)}/../lib/sawmill.rb")
 

data/tests/tc_multi_parser.rb

@@ -33,6 +33,7 @@
 # -----------------------------------------------------------------------------
 
 
+require 'rubygems'
 require 'test/unit'
 require 'stringio'
 require ::File.expand_path("#{::File.dirname(__FILE__)}/../lib/sawmill.rb")

data/tests/tc_record_processors.rb

@@ -33,6 +33,7 @@
 # -----------------------------------------------------------------------------
 
 
+require 'rubygems'
 require 'test/unit'
 require ::File.expand_path("#{::File.dirname(__FILE__)}/../lib/sawmill.rb")
 

data/tests/tc_records.rb

@@ -33,7 +33,9 @@
 # -----------------------------------------------------------------------------
 
 
+require 'rubygems'
 require 'test/unit'
+require 'set'
 require ::File.expand_path("#{::File.dirname(__FILE__)}/../lib/sawmill.rb")
 
 
@@ -108,7 +110,7 @@ module Sawmill
         record_ = @records.dequeue
         assert_equal(0, record_.message_count)
         assert_equal(5, record_.entry_count)
-        assert_equal(Set.new(['color', 'size']), Set.new(record_.attribute_keys))
+        assert_equal(::Set.new(['color', 'size']), ::Set.new(record_.attribute_keys))
         assert_equal('small', record_.attribute('size'))
         assert_equal('red', record_.attribute('color'))
       end
@@ -125,7 +127,7 @@ module Sawmill
         record_ = @records.dequeue
         assert_equal(0, record_.message_count)
         assert_equal(5, record_.entry_count)
-        assert_equal(Set.new(['color', 'size']), Set.new(record_.attribute_keys))
+        assert_equal(::Set.new(['color', 'size']), ::Set.new(record_.attribute_keys))
         assert_equal(['small'], record_.attribute('size'))
         assert_equal(['blue', 'red'], record_.attribute('color'))
       end

data/tests/tc_reports.rb

@@ -33,6 +33,7 @@
 # -----------------------------------------------------------------------------
 
 
+require 'rubygems'
 require 'test/unit'
 require 'stringio'
 require ::File.expand_path("#{::File.dirname(__FILE__)}/../lib/sawmill.rb")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: sawmill
 version: !ruby/object:Gem::Version 
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors: 
 - Daniel Azuma
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-06 00:00:00 -08:00
+date: 2009-11-08 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -20,17 +20,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.3.0
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: versionomy
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 0.2.0
+        version: 0.3.1
     version: 
 description: Sawmill is a logging and log analysis system for Ruby. It extends the basic Ruby logging facility with log records and parsing abilities.
 email: dazuma@gmail.com