fluent-plugin-parser 0.1.2 → 0.2.0

data/README.md

@@ -52,6 +52,19 @@ Format 'json' is also supported:
       key_name sales
     </match>
 
+Format 'ltsv'(Labeled-TSV (Tab separated values)) is also supported:
+
+    <match raw.sales.*>
+      type parser
+      tag sales
+      format ltsv
+      key_name sales
+    </match>
+
+'LTSV' is format like below, unlinke json, easy to write with simple formatter (ex: LogFormat of apache):
+
+    KEY1:VALUE1 [TAB] KEY2:VALUE2 [TAB] ...
+
 ### DeparserOutput
 
 To build CSV from field 'store','item','num', as field 'csv', without raw data:

data/fluent-plugin-parser.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-parser"
-  gem.version       = "0.1.2"
+  gem.version       = "0.2.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to parse single field, or to combine log structure into single field}

data/lib/fluent/plugin/fixed_parser.rb

@@ -47,15 +47,13 @@ class FluentExt::TextParser
     end
   end
 
-  class JSONParser
+  class GenericParser
     include Fluent::Configurable
 
     config_param :time_key, :string, :default => 'time'
     config_param :time_format, :string, :default => nil
 
-    def call(text)
-      record = Yajl.load(text)
-
+    def parse_time(record)
       time = nil
       
       if value = record.delete(@time_key)
@@ -67,15 +65,30 @@ class FluentExt::TextParser
       end
 
       return time, record
+    end
+  end
+
+  class JSONParser < GenericParser
+    def call(text)
+      record = Yajl.load(text)
+      return parse_time(record)
     rescue Yajl::ParseError
       return nil, nil
     end
   end
 
+  class LabeledTSVParser < GenericParser
+    def call(text)
+      record = Hash[text.split("\t").map{|p| p.split(":", 2)}]
+      parse_time(record)
+    end
+  end
+
   TEMPLATES = {
     'apache' => RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}),
     'syslog' => RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}),
     'json' => JSONParser.new,
+    'ltsv' => LabeledTSVParser.new,
   }
 
   def self.register_template(name, regexp_or_proc, time_format=nil)

data/test/plugin/test_out_parser.rb

@@ -51,6 +51,22 @@ class ParserOutputTest < Test::Unit::TestCase
         key_name foo
       ]
     }
+    assert_nothing_raised {
+      d = create_driver %[
+        remove_prefix foo.baz
+        add_prefix foo.bar
+        format json
+        key_name foo
+      ]
+    }
+    assert_nothing_raised {
+      d = create_driver %[
+        remove_prefix foo.baz
+        add_prefix foo.bar
+        format ltsv
+        key_name foo
+      ]
+    }
     d = create_driver %[
       tag foo.bar
       key_name foo
@@ -189,4 +205,60 @@ class ParserOutputTest < Test::Unit::TestCase
     assert_equal 'x', second[2]['xxx']
     assert_equal 'y', second[2]['yyy']
   end
+
+  CONFIG_LTSV =  %[
+    remove_prefix foo.baz
+    add_prefix foo.bar
+    format ltsv
+    key_name data
+  ]
+  def test_emit_ltsv
+    d = create_driver(CONFIG_LTSV, 'foo.baz.test')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.emit({'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'}, time)
+      d.emit({'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    emits = d.emits
+    assert_equal 2, emits.length
+
+    first = emits[0]
+    assert_equal 'foo.bar.test', first[0]
+    assert_equal time, first[1]
+    assert_nil first[2]['data']
+    assert_equal 'first', first[2]['xxx']
+    assert_equal 'second', first[2]['yyy']
+
+    second = emits[1]
+    assert_equal 'foo.bar.test', second[0]
+    assert_equal time, second[1]
+    assert_nil first[2]['data']
+    assert_equal 'first', second[2]['xxx']
+    assert_equal 'second2', second[2]['yyy']
+
+    d = create_driver(CONFIG_LTSV + %[
+      reserve_data yes
+    ], 'foo.baz.test')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.emit({'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'}, time)
+      d.emit({'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    emits = d.emits
+    assert_equal 2, emits.length
+
+    first = emits[0]
+    assert_equal 'foo.bar.test', first[0]
+    assert_equal time, first[1]
+    assert_equal "xxx:first\tyyy:second", first[2]['data']
+    assert_equal 'first', first[2]['xxx']
+    assert_equal 'second', first[2]['yyy']
+
+    second = emits[1]
+    assert_equal 'foo.bar.test', second[0]
+    assert_equal time, second[1]
+    assert_equal "xxx:first\tyyy:second", first[2]['data']
+    assert_equal 'first', second[2]['xxx']
+    assert_equal 'second2', second[2]['yyy']
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-12 00:00:00.000000000 Z
+date: 2013-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake