fluent-plugin-parser 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd40a834ddb84cf70b817ca1eeb890549786afdf
-  data.tar.gz: ec978eccafdcbde7569284036ef907b53cd0f2b9
+  metadata.gz: e389a96d9ae681aa25c0de95d9d1abb9b1c4189b
+  data.tar.gz: 3808413fd2cdd85483eeca9d01cdaed62e4fe1de
 SHA512:
-  metadata.gz: 5b8098ed0973e33d060dcdc6e07251d462cb70d79795b9abf1b294c89e7db4c9ac48ac5f2df11fa4df7c5fc521c6f6443aeae355cd5255bdd405be435430ceb2
-  data.tar.gz: a055940d6a07d97ba4e5438af542fb9cdf400f46115ce568eccdcb23497c3fcbf19bade3fae1a2e93e55f9c5f60d6073f2ea872a9c8947c3ca2642cce663d301
+  metadata.gz: 7174e622a987623dced94a830681e5d6f31742e2e2205964c709f9cccebca4a8a0b89b1ad082ded4cc9a8aedc8b71fe74e38af360bf33522476845639817d7c5
+  data.tar.gz: d6ea265808450c244ae34cc05efc62ef2e64a48b3b0c92327db373c49383d4d852b3ceb485dcf426d8200efc993fbf7ce3691ae8691f810296e85b19767781ef

data/fluent-plugin-parser.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-parser"
-  gem.version       = "0.5.0"
+  gem.version       = "0.6.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to parse single field, or to combine log structure into single field}

data/lib/fluent/plugin/filter_deparser.rb

@@ -0,0 +1,35 @@
+class Fluent::DeparserFilter < Fluent::Output
+  Fluent::Plugin.register_filter('deparser', self)
+
+  config_param :format, :string
+  config_param :format_key_names, :string
+  config_param :key_name, :string, default: 'message'
+  config_param :reserve_data, :bool, default: false
+
+  def configure(conf)
+    super
+
+    @format_key_names = @format_key_names.split(',')
+    begin
+      dummy = @format % (["x"] * @format_key_names.length)
+    rescue ArgumentError
+      raise Fluent::ConfigError, "mismatch between placeholder of format and format_key_names"
+    end
+  end
+
+  def filter_stream(tag, es)
+    new_es = Fluent::MultiEventStream.new
+    if @reserve_data
+      es.each {|time,record|
+        new_record = {@key_name => (@format % @format_key_names.map{|k| record[k]})}
+        new_es.add(time, record.merge(new_record))
+      }
+    else
+      es.each {|time,record|
+        new_record = {@key_name => (@format % @format_key_names.map{|k| record[k]})}
+        new_es.add(time, new_record)
+      }
+    end
+    new_es
+  end
+end

data/lib/fluent/plugin/filter_parser.rb

@@ -0,0 +1,105 @@
+require 'fluent/parser'
+
+class Fluent::ParserFilter < Fluent::Filter
+  Fluent::Plugin.register_filter('parser', self)
+
+  config_param :key_name, :string
+  config_param :reserve_data, :bool, default: false
+  config_param :inject_key_prefix, :string, default: nil
+  config_param :replace_invalid_sequence, :bool, default: false
+  config_param :hash_value_field, :string, default: nil
+  config_param :suppress_parse_error_log, :bool, default: false
+  config_param :time_parse, :bool, default: true
+
+  attr_reader :parser
+
+  def initialize
+    super
+    require 'time'
+  end
+
+  def configure(conf)
+    super
+
+    @parser = Fluent::TextParser.new
+    @parser.estimate_current_event = false
+    @parser.configure(conf)
+    if !@time_parse && @parser.parser.respond_to?("time_key=".to_sym)
+      # disable parse time
+      @parser.parser.time_key = nil
+    end
+
+    self
+  end
+
+  def filter_stream(tag, es)
+    new_es = Fluent::MultiEventStream.new
+    es.each do |time,record|
+      raw_value = record[@key_name]
+      begin
+        @parser.parse(raw_value) do |t,values|
+          if values
+            t ||= time
+            r = handle_parsed(tag, record, t, values)
+            new_es.add(t, r)
+          else
+            log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
+            if @reserve_data
+              t = time
+              r = handle_parsed(tag, record, time, {})
+              new_es.add(t, r)
+            end
+          end
+        end
+      rescue Fluent::TextParser::ParserError => e
+        log.warn e.message unless @suppress_parse_error_log
+      rescue ArgumentError => e
+        if @replace_invalid_sequence
+          unless e.message.index("invalid byte sequence in") == 0
+            raise
+          end
+          replaced_string = replace_invalid_byte(raw_value)
+          @parser.parse(replaced_string) do |t,values|
+            if values
+              t ||= time
+              r = handle_parsed(tag, record, t, values)
+              new_es.add(t, r)
+            else
+              log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
+              if @reserve_data
+                t = time
+                r = handle_parsed(tag, record, time, {})
+                new_es.add(t, r)
+              end
+            end
+          end
+        else
+          raise
+        end
+      rescue => e
+        log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
+      end
+    end
+    new_es
+  end
+
+  private
+
+  def handle_parsed(tag, record, t, values)
+    if values && @inject_key_prefix
+      values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
+    end
+    r = @hash_value_field ? {@hash_value_field => values} : values
+    if @reserve_data
+      r = r ? record.merge(r) : record
+    end
+    r
+  end
+
+  def replace_invalid_byte(string)
+    replace_options = { invalid: :replace, undef: :replace, replace: '?' }
+    original_encoding = string.encoding
+    temporal_encoding = (original_encoding == Encoding::UTF_8 ? Encoding::UTF_16BE : Encoding::UTF_8)
+    string.encode(temporal_encoding, original_encoding, replace_options).encode(original_encoding)
+  end
+end

data/test/helper.rb

@@ -24,6 +24,8 @@ end
 
 require 'fluent/plugin/out_parser'
 require 'fluent/plugin/out_deparser'
+require 'fluent/plugin/filter_parser'
+require 'fluent/plugin/filter_deparser'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_filter_deparser.rb

@@ -0,0 +1,82 @@
+require 'helper'
+
+class DeparserFilterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    format        %s: %s %s %s
+    format_key_names host,path,status,size
+    key_name      fulltext
+    reserve_data  true
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::FilterTestDriver.new(Fluent::DeparserFilter, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+
+    d = create_driver %[
+      format %s: %s %s
+      format_key_names x,y,z
+    ]
+    assert_equal '%s: %s %s', d.instance.format
+    assert_equal ['x','y','z'], d.instance.format_key_names
+    assert_equal 'message', d.instance.key_name
+    assert_equal false, d.instance.reserve_data
+  end
+
+  # CONFIG = %[
+  #   format        %s: %s %s %s
+  #   format_key_names host path status size
+  #   key_name      fulltext
+  #   reserve_data  true
+  # ]
+  def test_filter
+    d1 = create_driver(CONFIG, 'test.no.change')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d1.run do
+      d1.filter({'host'=>'xxx.local','path'=>'/f/1','status'=>'200','size'=>300}, time)
+      d1.filter({'host'=>'yyy.local','path'=>'/f/2','status'=>'302','size'=>512}, time)
+    end
+    filtered = d1.filtered_as_array
+    assert_equal 2, filtered.length
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_equal 'xxx.local: /f/1 200 300', first[2]['fulltext']
+    assert_equal ['fulltext','host','path','size','status'], first[2].keys.sort
+    second = filtered[1]
+    assert_equal 'test.no.change', second[0]
+    assert_equal time, second[1]
+    assert_equal 'yyy.local: /f/2 302 512', second[2]['fulltext']
+    assert_equal ['fulltext','host','path','size','status'], second[2].keys.sort
+
+    d2 = create_driver(%[
+      format %s: %s %s
+      format_key_names host,path,status
+    ], 'test.no.change')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d2.run do
+      d2.filter({'host'=>'xxx.local','path'=>'/f/1','status'=>'200','size'=>300}, time)
+      d2.filter({'host'=>'yyy.local','path'=>'/f/2','status'=>'302','size'=>512}, time)
+    end
+    filtered = d2.filtered_as_array
+    assert_equal 2, filtered.length
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_equal 'xxx.local: /f/1 200', first[2]['message']
+    assert_equal ['message'], first[2].keys.sort
+    second = filtered[1]
+    assert_equal 'test.no.change', second[0]
+    assert_equal time, second[1]
+    assert_equal 'yyy.local: /f/2 302', second[2]['message']
+    assert_equal ['message'], second[2].keys.sort
+  end
+end

data/test/plugin/test_filter_parser.rb

@@ -0,0 +1,668 @@
+require 'helper'
+
+class ParserFilterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    key_name      message
+    format        /^(?<x>.)(?<y>.) (?<time>.+)$/
+    time_format   %Y%m%d%H%M%S
+    reserve_data  true
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::FilterTestDriver.new(Fluent::ParserFilter, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver %[
+        format unknown_format_that_will_never_be_implemented
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format /(?<x>.)/
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format /(?<x>.)/
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format /(?<x>.)/
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format /(?<x>.)/
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format json
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format ltsv
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+    d = create_driver %[
+        format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+        key_name message
+        suppress_parse_error_log true
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+        key_name message
+        suppress_parse_error_log false
+      ]
+    }
+    d = create_driver %[
+      key_name foo
+      format /(?<x>.)/
+    ]
+    assert_equal false, d.instance.reserve_data
+  end
+
+  # CONFIG = %[
+  #   remove_prefix test
+  #   add_prefix    parsed
+  #   key_name      message
+  #   format        /^(?<x>.)(?<y>.) (?<time>.+)$/
+  #   time_format   %Y%m%d%H%M%S
+  #   reserve_data  true
+  # ]
+  def test_filter
+    d1 = create_driver(CONFIG, 'test.no.change')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d1.run do
+      d1.filter({'message' => '12 20120402182059'}, time)
+      d1.filter({'message' => '34 20120402182100'}, time)
+      d1.filter({'message' => '56 20120402182100'}, time)
+      d1.filter({'message' => '78 20120402182101'}, time)
+      d1.filter({'message' => '90 20120402182100'}, time)
+    end
+    filtered = d1.filtered_as_array
+    assert_equal 5, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal Time.parse("2012-04-02 18:20:59").to_i, first[1]
+    assert_equal '1', first[2]['x']
+    assert_equal '2', first[2]['y']
+    assert_equal '12 20120402182059', first[2]['message']
+
+    second = filtered[1]
+    assert_equal 'test.no.change', second[0]
+    assert_equal Time.parse("2012-04-02 18:21:00").to_i, second[1]
+    assert_equal '3', second[2]['x']
+    assert_equal '4', second[2]['y']
+
+    third = filtered[2]
+    assert_equal 'test.no.change', third[0]
+    assert_equal Time.parse("2012-04-02 18:21:00").to_i, third[1]
+    assert_equal '5', third[2]['x']
+    assert_equal '6', third[2]['y']
+
+    fourth = filtered[3]
+    assert_equal 'test.no.change', fourth[0]
+    assert_equal Time.parse("2012-04-02 18:21:01").to_i, fourth[1]
+    assert_equal '7', fourth[2]['x']
+    assert_equal '8', fourth[2]['y']
+
+    fifth = filtered[4]
+    assert_equal 'test.no.change', fifth[0]
+    assert_equal Time.parse("2012-04-02 18:21:00").to_i, fifth[1]
+    assert_equal '9', fifth[2]['x']
+    assert_equal '0', fifth[2]['y']
+
+    d2 = create_driver(%[
+      tag parsed
+      key_name      data
+      format        /^(?<x>.)(?<y>.) (?<t>.+)$/
+    ], 'test.no.change')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d2.run do
+      d2.filter({'data' => '12 20120402182059'}, time)
+      d2.filter({'data' => '34 20120402182100'}, time)
+    end
+    filtered = d2.filtered_as_array
+    assert_equal 2, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_nil first[2]['data']
+    assert_equal '1', first[2]['x']
+    assert_equal '2', first[2]['y']
+    assert_equal '20120402182059', first[2]['t']
+
+    second = filtered[1]
+    assert_equal 'test.no.change', second[0]
+    assert_equal time, second[1]
+    assert_nil second[2]['data']
+    assert_equal '3', second[2]['x']
+    assert_equal '4', second[2]['y']
+    assert_equal '20120402182100', second[2]['t']
+
+    d3 = create_driver(%[
+      tag parsed
+      key_name      data
+      format        /^(?<x>[0-9])(?<y>[0-9]) (?<t>.+)$/
+    ], 'test.no.change')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d3.run do
+      d3.filter({'data' => '12 20120402182059'}, time)
+      d3.filter({'data' => '34 20120402182100'}, time)
+      d3.filter({'data' => 'xy 20120402182101'}, time)
+    end
+    filtered = d3.filtered_as_array
+    assert_equal 2, filtered.length
+
+    d3x = create_driver(%[
+      tag parsed
+      key_name      data
+      format        /^(?<x>\\d)(?<y>\\d) (?<t>.+)$/
+      reserve_data  yes
+    ], 'test.no.change')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d3x.run do
+      d3x.filter({'data' => '12 20120402182059'}, time)
+      d3x.filter({'data' => '34 20120402182100'}, time)
+      d3x.filter({'data' => 'xy 20120402182101'}, time)
+    end
+    filtered = d3x.filtered_as_array
+    assert_equal 3, filtered.length
+
+    d4 = create_driver(%[
+      tag parsed
+      key_name      data
+      format        json
+    ], 'test.no.change')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d4.run do
+      d4.filter({'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}, time)
+      d4.filter({'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    filtered = d4.filtered_as_array
+    assert_equal 1, filtered.length
+
+    d4x = create_driver(%[
+      tag parsed
+      key_name      data
+      format        json
+      reserve_data  yes
+    ], 'test.no.change')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d4x.run do
+      d4x.filter({'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}, time)
+      d4x.filter({'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    filtered = d4x.filtered_as_array
+    assert_equal 2, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_equal '{"xxx":"first","yyy":"second"}', first[2]['data']
+    assert_equal 'first', first[2]['xxx']
+    assert_equal 'second', first[2]['yyy']
+
+    second = filtered[1]
+    assert_equal 'test.no.change', second[0]
+    assert_equal time, second[1]
+    assert_equal 'foobar', second[2]['data']
+    assert_equal 'x', second[2]['xxx']
+    assert_equal 'y', second[2]['yyy']
+  end
+
+  CONFIG_LTSV =  %[
+    format ltsv
+    key_name data
+  ]
+  def test_filter_ltsv
+    d = create_driver(CONFIG_LTSV, 'test.no.change')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.filter({'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'}, time)
+      d.filter({'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 2, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_nil first[2]['data']
+    assert_equal 'first', first[2]['xxx']
+    assert_equal 'second', first[2]['yyy']
+
+    second = filtered[1]
+    assert_equal 'test.no.change', second[0]
+    assert_equal time, second[1]
+    assert_nil first[2]['data']
+    assert_equal 'first', second[2]['xxx']
+    assert_equal 'second2', second[2]['yyy']
+
+    d = create_driver(CONFIG_LTSV + %[
+      reserve_data yes
+    ], 'test.no.change')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.filter({'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'}, time)
+      d.filter({'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 2, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_equal "xxx:first\tyyy:second", first[2]['data']
+    assert_equal 'first', first[2]['xxx']
+    assert_equal 'second', first[2]['yyy']
+
+    second = filtered[1]
+    assert_equal 'test.no.change', second[0]
+    assert_equal time, second[1]
+    assert_equal "xxx:first\tyyy:second", first[2]['data']
+    assert_equal 'first', second[2]['xxx']
+    assert_equal 'second2', second[2]['yyy']
+
+    # convert types
+    d = create_driver(CONFIG_LTSV + %[
+      types i:integer,s:string,f:float,b:bool
+    ], 'test.no.change')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.filter({'data' => "i:1\ts:2\tf:3\tb:true\tx:123"}, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_equal 1, first[2]['i']
+    assert_equal '2', first[2]['s']
+    assert_equal 3.0, first[2]['f']
+    assert_equal true, first[2]['b']
+    assert_equal '123', first[2]['x']
+  end
+
+  CONFIG_TSV =  %[
+    format tsv
+    key_name data
+    keys key1,key2,key3
+  ]
+  def test_filter_tsv
+    d = create_driver(CONFIG_TSV, 'test.no.change')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.filter({'data' => "value1\tvalue2\tvalueThree", 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_nil first[2]['data']
+    assert_equal 'value1', first[2]['key1']
+    assert_equal 'value2', first[2]['key2']
+    assert_equal 'valueThree', first[2]['key3']
+  end
+
+  CONFIG_CSV =  %[
+    format csv
+    key_name data
+    keys key1,key2,key3
+  ]
+  def test_filter_csv
+    d = create_driver(CONFIG_CSV, 'test.no.change')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.filter({'data' => 'value1,"value2","value""ThreeYes!"', 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+    assert_nil first[2]['data']
+    assert_equal 'value1', first[2]['key1']
+    assert_equal 'value2', first[2]['key2']
+    assert_equal 'value"ThreeYes!', first[2]['key3']
+  end
+
+  CONFIG_HASH_VALUE_FIELD = %[
+    format       json
+    key_name     data
+    hash_value_field parsed
+  ]
+  CONFIG_HASH_VALUE_FIELD_RESERVE_DATA = %[
+    format       json
+    key_name     data
+    reserve_data yes
+    hash_value_field parsed
+  ]
+  CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX = %[
+    format       json
+    key_name     data
+    hash_value_field parsed
+    inject_key_prefix data.
+  ]
+  def test_filter_inject_hash_value_field
+    original = {'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}
+
+    d = create_driver(CONFIG_HASH_VALUE_FIELD, 'test.no.change')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.filter(original, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+
+    record = first[2]
+    assert_equal 1, record.keys.size
+    assert_equal({"xxx"=>"first","yyy"=>"second"}, record['parsed'])
+
+    d = create_driver(CONFIG_HASH_VALUE_FIELD_RESERVE_DATA, 'test.no.change')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.filter(original, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+
+    record = first[2]
+    assert_equal 4, record.keys.size
+    assert_equal original['data'], record['data']
+    assert_equal original['xxx'], record['xxx']
+    assert_equal original['yyy'], record['yyy']
+    assert_equal({"xxx"=>"first","yyy"=>"second"}, record['parsed'])
+
+    d = create_driver(CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX, 'test.no.change')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.filter(original, time)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    first = filtered[0]
+    assert_equal 'test.no.change', first[0]
+    assert_equal time, first[1]
+
+    record = first[2]
+    assert_equal 1, record.keys.size
+    assert_equal({"data.xxx"=>"first","data.yyy"=>"second"}, record['parsed'])
+  end
+
+  CONFIG_DONT_PARSE_TIME = %[
+    key_name data
+    format json
+    time_parse no
+  ]
+  def test_time_should_be_reserved
+    t = Time.now.to_i
+    d = create_driver(CONFIG_DONT_PARSE_TIME, 'test.no.change')
+
+    d.run do
+      d.filter({'data' => '{"time":1383190430, "f1":"v1"}'}, t)
+      d.filter({'data' => '{"time":"1383190430", "f1":"v1"}'}, t)
+      d.filter({'data' => '{"time":"2013-10-31 12:34:03 +0900", "f1":"v1"}'}, t)
+    end
+    filtered = d.filtered_as_array
+    assert_equal 3, filtered.length
+
+    assert_equal 'test.no.change', filtered[0][0]
+    assert_equal 'v1', filtered[0][2]['f1']
+    assert_equal 1383190430, filtered[0][2]['time']
+    assert_equal t, filtered[0][1]
+
+    assert_equal 'test.no.change', filtered[1][0]
+    assert_equal 'v1', filtered[1][2]['f1']
+    assert_equal "1383190430", filtered[1][2]['time']
+    assert_equal t, filtered[1][1]
+
+    assert_equal 'test.no.change', filtered[2][0]
+    assert_equal 'v1', filtered[2][2]['f1']
+    assert_equal '2013-10-31 12:34:03 +0900', filtered[2][2]['time']
+    assert_equal t, filtered[2][1]
+  end
+
+  CONFIG_INVALID_TIME_VALUE = %[
+    remove_prefix test
+    key_name data
+    format json
+  ] # 'time' is implicit @time_key
+  def test_filter_invalid_time_data
+    # should not raise errors
+    t = Time.now.to_i
+    d = create_driver(CONFIG_INVALID_TIME_VALUE, 'test.no.change')
+    assert_nothing_raised {
+      d.run do
+        d.filter({'data' => '{"time":[], "f1":"v1"}'}, t)
+        d.filter({'data' => '{"time":"thisisnottime", "f1":"v1"}'}, t)
+      end
+    }
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+
+    assert_equal 'test.no.change', filtered[0][0]
+    assert_equal 0, filtered[0][1]
+    assert_equal 'v1', filtered[0][2]['f1']
+    assert_equal 0, filtered[0][2]['time'].to_i
+  end
+
+  # REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+
+  CONFIG_NOT_REPLACE = %[
+    remove_prefix test
+    key_name      data
+    format        /^(?<message>.*)$/
+  ]
+  CONFIG_INVALID_BYTE = CONFIG_NOT_REPLACE + %[
+    replace_invalid_sequence true
+  ]
+  def test_filter_invalid_byte
+    invalid_utf8 = "\xff".force_encoding('UTF-8')
+
+    d = create_driver(CONFIG_NOT_REPLACE, 'test.no.change')
+    assert_raise(ArgumentError) {
+      d.run do
+        d.filter({'data' => invalid_utf8}, Time.now.to_i)
+      end
+    }
+
+    d = create_driver(CONFIG_INVALID_BYTE, 'test.in')
+    assert_nothing_raised {
+      d.run do
+        d.emit({'data' => invalid_utf8}, Time.now.to_i)
+      end
+    }
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+    assert_nil filtered[0][2]['data']
+    assert_equal '?'.force_encoding('UTF-8'), filtered[0][2]['message']
+
+    d = create_driver(CONFIG_INVALID_BYTE + %[
+      reserve_data yes
+    ], 'test.no.change')
+    assert_nothing_raised {
+      d.run do
+        d.filter({'data' => invalid_utf8}, Time.now.to_i)
+      end
+    }
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+    assert_equal invalid_utf8, filtered[0][2]['data']
+    assert_equal '?'.force_encoding('UTF-8'), filtered[0][2]['message']
+
+    invalid_ascii = "\xff".force_encoding('US-ASCII')
+    d = create_driver(CONFIG_INVALID_BYTE, 'test.no.change')
+    assert_nothing_raised {
+      d.run do
+        d.filter({'data' => invalid_ascii}, Time.now.to_i)
+      end
+    }
+    filtered = d.filtered_as_array
+    assert_equal 1, filtered.length
+    assert_nil filtered[0][2]['data']
+    assert_equal '?'.force_encoding('US-ASCII'), filtered[0][2]['message']
+  end
+
+  # suppress_parse_error_log test
+  CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG = %[
+    tag hogelog
+    format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+    key_name message
+    suppress_parse_error_log false
+  ]
+  CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG = %[
+    tag hogelog
+    format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+    key_name message
+    suppress_parse_error_log true
+  ]
+  CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG = %[
+    tag hogelog
+    format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
+    key_name message
+  ]
+
+  INVALID_MESSAGE = 'foo bar'
+  VALID_MESSAGE   = 'col1=foo col2=bar'
+
+  # if call warn() raise exception
+  class DummyLoggerWarnedException < StandardError; end
+  class DummyLogger
+    def warn(message)
+      raise DummyLoggerWarnedException
+    end
+  end
+
+  def swap_logger(instance)
+    raise "use with block" unless block_given?
+    dummy = DummyLogger.new
+    saved_logger = instance.log
+    instance.log = dummy
+    restore = lambda{ instance.log = saved_logger }
+
+    yield
+
+    restore.call
+  end
+
+  def test_parser_error_warning
+    d = create_driver(CONFIG_INVALID_TIME_VALUE, 'test.no.change')
+    swap_logger(d.instance) do
+      assert_raise(DummyLoggerWarnedException) {
+        d.run do
+          d.filter({'data' => '{"time":[], "f1":"v1"}'}, Time.now.to_i)
+        end
+      }
+    end
+  end
+
+  class DefaultSuppressParseErrorLogTest < self
+    def setup
+      # default(disabled) 'suppress_parse_error_log' is not specify
+      @d = create_driver(CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG, 'test.no.change')
+    end
+
+    def test_raise_exception
+      swap_logger(@d.instance) do
+        assert_raise(DummyLoggerWarnedException) {
+          @d.run do
+            @d.filter({'message' => INVALID_MESSAGE}, Time.now.to_i)
+          end
+        }
+      end
+    end
+
+    def test_nothing_raised
+      swap_logger(@d.instance) do
+      assert_nothing_raised {
+        @d.run do
+          @d.filter({'message' => VALID_MESSAGE}, Time.now.to_i)
+        end
+      }
+      end
+    end
+  end
+
+  class DisabledSuppressParseErrorLogTest < self
+    def setup
+      # disabled 'suppress_parse_error_log'
+      @d = create_driver(CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.no.change')
+    end
+
+    def test_raise_exception
+      swap_logger(@d.instance) do
+        assert_raise(DummyLoggerWarnedException) {
+          @d.run do
+            @d.filter({'message' => INVALID_MESSAGE}, Time.now.to_i)
+          end
+        }
+      end
+    end
+
+    def test_nothing_raised
+      swap_logger(@d.instance) do
+      assert_nothing_raised {
+        @d.run do
+          @d.filter({'message' => VALID_MESSAGE}, Time.now.to_i)
+        end
+      }
+      end
+    end
+  end
+
+  class EnabledSuppressParseErrorLogTest < self
+    def setup
+      # enabled 'suppress_parse_error_log'
+      @d = create_driver(CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.no.change')
+    end
+
+    def test_nothing_raised
+      swap_logger(@d.instance) do
+      assert_nothing_raised {
+        @d.run do
+          @d.filter({'message' => INVALID_MESSAGE}, Time.now.to_i)
+          @d.filter({'message' => VALID_MESSAGE},   Time.now.to_i)
+        end
+      }
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-04 00:00:00.000000000 Z
+date: 2015-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit
@@ -67,11 +67,15 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-parser.gemspec
+- lib/fluent/plugin/filter_deparser.rb
+- lib/fluent/plugin/filter_parser.rb
 - lib/fluent/plugin/out_deparser.rb
 - lib/fluent/plugin/out_parser.rb
 - test/custom_parser.rb
 - test/helper.rb
 - test/plugin/test_deparser.rb
+- test/plugin/test_filter_deparser.rb
+- test/plugin/test_filter_parser.rb
 - test/plugin/test_out_parser.rb
 - test/plugin/test_out_parser_for_parsers.rb
 homepage: https://github.com/tagomoris/fluent-plugin-parser
@@ -102,5 +106,8 @@ test_files:
 - test/custom_parser.rb
 - test/helper.rb
 - test/plugin/test_deparser.rb
+- test/plugin/test_filter_deparser.rb
+- test/plugin/test_filter_parser.rb
 - test/plugin/test_out_parser.rb
 - test/plugin/test_out_parser_for_parsers.rb
+has_rdoc: