fluent-plugin-parser 0.2.1 → 0.2.2

data/README.md

@@ -65,6 +65,20 @@ Format 'ltsv'(Labeled-TSV (Tab separated values)) is also supported:
 
     KEY1:VALUE1 [TAB] KEY2:VALUE2 [TAB] ...
 
+About LTSV, see: http://ltsv.org/
+
+If you want to suppress 'pattern not match' log, specify 'suppress_parse_error_log true' to configuration.
+default value is false.
+
+    <match in.hogelog>
+      type parser
+      tag hogelog
+      format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+      key_name message
+      suppress_parse_error_log true
+    </match>
+
+
 ### DeparserOutput
 
 To build CSV from field 'store','item','num', as field 'csv', without raw data:

data/fluent-plugin-parser.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-parser"
-  gem.version       = "0.2.1"
+  gem.version       = "0.2.2"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to parse single field, or to combine log structure into single field}

data/lib/fluent/plugin/fixed_parser.rb

@@ -8,6 +8,7 @@ class FluentExt::TextParser
     include Fluent::Configurable
 
     config_param :time_format, :string, :default => nil
+    config_param :suppress_parse_error_log, :bool, :default => false
 
     def initialize(regexp, conf={})
       super()
@@ -20,7 +21,10 @@ class FluentExt::TextParser
     def call(text)
       m = @regexp.match(text)
       unless m
-        $log.warn "pattern not match: #{text}"
+        unless @suppress_parse_error_log
+          $log.warn "pattern not match: #{text}"
+        end
+
         return nil, nil
       end
 
@@ -70,7 +74,10 @@ class FluentExt::TextParser
       record = Yajl.load(text)
       return parse_time(record)
     rescue Yajl::ParseError
-      $log.warn "pattern not match(json): #{text.inspect}: #{$!}"
+      unless @suppress_parse_error_log
+        $log.warn "pattern not match(json): #{text.inspect}: #{$!}"
+      end
+
       return nil, nil
     end
   end
@@ -122,7 +129,10 @@ class FluentExt::TextParser
     def call(text)
       m = REGEXP.match(text)
       unless m
-        $log.warn "pattern not match: #{text.inspect}"
+        unless @suppress_parse_error_log
+          $log.warn "pattern not match: #{text.inspect}"
+        end
+
         return nil, nil
       end
 

data/lib/fluent/plugin/out_parser.rb

@@ -8,6 +8,7 @@ class Fluent::ParserOutput < Fluent::Output
   config_param :add_prefix, :string, :default => nil
   config_param :key_name, :string
   config_param :reserve_data, :bool, :default => false
+  config_param :replace_invalid_sequence, :bool, :default => false
 
   def initialize
     super
@@ -56,7 +57,7 @@ class Fluent::ParserOutput < Fluent::Output
       es.each {|time,record|
         value = record[@key_name]
         t,values = if value
-                     @parser.parse(value)
+                     parse(value)
                    else
                      [nil, nil]
                    end
@@ -72,7 +73,7 @@ class Fluent::ParserOutput < Fluent::Output
       es.each {|time,record|
         value = record[@key_name]
         t,values = if value
-                     @parser.parse(value)
+                     parse(value)
                    else
                      [nil, nil]
                    end
@@ -84,4 +85,27 @@ class Fluent::ParserOutput < Fluent::Output
     end
     chain.next
   end
+
+  private
+
+  def parse(string)
+    return @parser.parse(string) unless @replace_invalid_sequence
+
+    begin
+      @parser.parse(string)
+    rescue ArgumentError => e
+      unless e.message.index("invalid byte sequence in") == 0
+        raise
+      end
+      replaced_string = replace_invalid_byte(string)
+      @parser.parse(replaced_string)
+    end
+  end
+
+  def replace_invalid_byte(string)
+    replace_options = { invalid: :replace, undef: :replace, replace: '?' }
+    original_encoding = string.encoding
+    temporal_encoding = (original_encoding == Encoding::UTF_8 ? Encoding::UTF_16BE : Encoding::UTF_8)
+    string.encode(temporal_encoding, original_encoding, replace_options).encode(original_encoding)
+  end
 end

data/test/plugin/test_out_parser.rb

@@ -67,6 +67,22 @@ class ParserOutputTest < Test::Unit::TestCase
         key_name foo
       ]
     }
+    assert_nothing_raised {
+    d = create_driver %[
+        tag hogelog
+        format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+        key_name message
+        suppress_parse_error_log true
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        tag hogelog
+        format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+        key_name message
+        suppress_parse_error_log false
+      ]
+    }
     d = create_driver %[
       tag foo.bar
       key_name foo
@@ -315,4 +331,136 @@ class ParserOutputTest < Test::Unit::TestCase
   #TODO: apache2
   # REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
 
+  CONFIG_NOT_REPLACE = %[
+    remove_prefix test
+    key_name      data
+    format        /^(?<message>.*)$/
+  ]
+  CONFIG_INVALID_BYTE = CONFIG_NOT_REPLACE + %[
+    replace_invalid_sequence true
+  ]
+  def test_emit_invalid_byte
+    invalid_utf8 = "\xff".force_encoding('UTF-8')
+
+    d = create_driver(CONFIG_NOT_REPLACE, 'test.in')
+    assert_raise(ArgumentError) {
+      d.run do
+        d.emit({'data' => invalid_utf8}, Time.now.to_i)
+      end
+    }
+
+    d = create_driver(CONFIG_INVALID_BYTE, 'test.in')
+    assert_nothing_raised {
+      d.run do
+        d.emit({'data' => invalid_utf8}, Time.now.to_i)
+      end
+    }
+    emits = d.emits
+    assert_equal 1, emits.length
+    assert_nil emits[0][2]['data']
+    assert_equal '?'.force_encoding('UTF-8'), emits[0][2]['message']
+
+    d = create_driver(CONFIG_INVALID_BYTE + %[
+      reserve_data yes
+    ], 'test.in')
+    assert_nothing_raised {
+      d.run do
+        d.emit({'data' => invalid_utf8}, Time.now.to_i)
+      end
+    }
+    emits = d.emits
+    assert_equal 1, emits.length
+    assert_equal invalid_utf8, emits[0][2]['data']
+    assert_equal '?'.force_encoding('UTF-8'), emits[0][2]['message']
+
+    invalid_ascii = "\xff".force_encoding('US-ASCII')
+    d = create_driver(CONFIG_INVALID_BYTE, 'test.in')
+    assert_nothing_raised {
+      d.run do
+        d.emit({'data' => invalid_ascii}, Time.now.to_i)
+      end
+    }
+    emits = d.emits
+    assert_equal 1, emits.length
+    assert_nil emits[0][2]['data']
+    assert_equal '?'.force_encoding('US-ASCII'), emits[0][2]['message']
+  end
+
+  # suppress_parse_error_log test 
+  CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG = %[
+    tag hogelog
+    format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+    key_name message
+    suppress_parse_error_log false 
+  ]
+  CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG = %[
+    tag hogelog
+    format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+    key_name message
+    suppress_parse_error_log true 
+  ]
+  CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG = %[
+    tag hogelog
+    format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+    key_name message
+  ]
+
+  INVALID_MESSAGE = 'foo bar'
+  VALID_MESSAGE   = 'col1=foo col2=bar'
+
+  # if call warn() raise exception
+  class DummyLoggerWarnedException < StandardError; end
+  class DummyLogger
+    def warn(message)
+      raise DummyLoggerWarnedException
+    end
+  end
+
+  def test_suppress_parse_error_log
+    # default(disabled) 'suppress_parse_error_log' is not specify
+    d = create_driver(CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
+
+    saved_logger = $log
+    $log = DummyLogger.new
+
+    assert_raise(DummyLoggerWarnedException) {
+      d.run do
+        d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
+      end
+    }
+
+    assert_nothing_raised {
+      d.run do
+        d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
+      end
+    }
+
+    # disabled 'suppress_parse_error_log'
+    d = create_driver(CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
+
+    assert_raise(DummyLoggerWarnedException) {
+      d.run do
+        d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
+      end
+    }
+
+    assert_nothing_raised {
+      d.run do
+        d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
+      end
+    }
+
+    # enabled 'suppress_parse_error_log'
+    d = create_driver(CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
+
+    assert_nothing_raised {
+      d.run do
+        d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
+        d.emit({'message' => VALID_MESSAGE},   Time.now.to_i)
+      end
+    }
+
+    $log = saved_logger
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-07 00:00:00.000000000 Z
+date: 2013-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake