fluent-plugin-parser 0.1.1 → 0.1.2

data/README.md

@@ -43,6 +43,15 @@ If you want original attribute-data pair in re-emitted message, specify 'reserve
       reserve_data yes
     </match>
 
+Format 'json' is also supported:
+
+    <match raw.sales.*>
+      type parser
+      tag sales
+      format json
+      key_name sales
+    </match>
+
 ### DeparserOutput
 
 To build CSV from field 'store','item','num', as field 'csv', without raw data:

data/fluent-plugin-parser.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-parser"
-  gem.version       = "0.1.1"
+  gem.version       = "0.1.2"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to parse single field, or to combine log structure into single field}

data/lib/fluent/plugin/fixed_parser.rb

@@ -67,6 +67,8 @@ class FluentExt::TextParser
       end
 
       return time, record
+    rescue Yajl::ParseError
+      return nil, nil
     end
   end
 

data/lib/fluent/plugin/out_parser.rb

@@ -58,11 +58,15 @@ class Fluent::ParserOutput < Fluent::Output
         t,values = if value
                      @parser.parse(value)
                    else
-                     [nil, {}]
+                     [nil, nil]
                    end
         t ||= time
-        record.update(values)
-        Fluent::Engine.emit(tag, t, record)
+        r = if values
+              record.merge(values)
+            else
+              record
+            end
+        Fluent::Engine.emit(tag, t, r)
       }
     else
       es.each {|time,record|
@@ -70,10 +74,12 @@ class Fluent::ParserOutput < Fluent::Output
         t,values = if value
                      @parser.parse(value)
                    else
-                     [nil, {}]
+                     [nil, nil]
                    end
         t ||= time
-        Fluent::Engine.emit(tag, t, values)
+        if values
+          Fluent::Engine.emit(tag, t, values)
+        end
       }
     end
     chain.next

data/test/plugin/test_out_parser.rb

@@ -118,5 +118,75 @@ class ParserOutputTest < Test::Unit::TestCase
     assert_equal '3', second[2]['x']
     assert_equal '4', second[2]['y']
     assert_equal '20120402182100', second[2]['t']
+
+    d3 = create_driver(%[
+      tag parsed
+      key_name      data
+      format        /^(?<x>[0-9])(?<y>[0-9]) (?<t>.+)$/
+    ], 'test.in')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d3.run do
+      d3.emit({'data' => '12 20120402182059'}, time)
+      d3.emit({'data' => '34 20120402182100'}, time)
+      d3.emit({'data' => 'xy 20120402182101'}, time)
+    end
+    emits = d3.emits
+    assert_equal 2, emits.length
+
+    d3x = create_driver(%[
+      tag parsed
+      key_name      data
+      format        /^(?<x>\d)(?<y>\d) (?<t>.+)$/
+      reserve_data  yes
+    ], 'test.in')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d3x.run do
+      d3x.emit({'data' => '12 20120402182059'}, time)
+      d3x.emit({'data' => '34 20120402182100'}, time)
+      d3x.emit({'data' => 'xy 20120402182101'}, time)
+    end
+    emits = d3x.emits
+    assert_equal 3, emits.length
+
+    d4 = create_driver(%[
+      tag parsed
+      key_name      data
+      format        json
+    ], 'test.in')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d4.run do
+      d4.emit({'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}, time)
+      d4.emit({'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    emits = d4.emits
+    assert_equal 1, emits.length
+
+    d4x = create_driver(%[
+      tag parsed
+      key_name      data
+      format        json
+      reserve_data  yes
+    ], 'test.in')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d4x.run do
+      d4x.emit({'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}, time)
+      d4x.emit({'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    emits = d4x.emits
+    assert_equal 2, emits.length
+
+    first = emits[0]
+    assert_equal 'parsed', first[0]
+    assert_equal time, first[1]
+    assert_equal '{"xxx":"first","yyy":"second"}', first[2]['data']
+    assert_equal 'first', first[2]['xxx']
+    assert_equal 'second', first[2]['yyy']
+
+    second = emits[1]
+    assert_equal 'parsed', second[0]
+    assert_equal time, second[1]
+    assert_equal 'foobar', second[2]['data']
+    assert_equal 'x', second[2]['xxx']
+    assert_equal 'y', second[2]['yyy']
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors: