fluent-plugin-parser 0.3.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 11c8d0bf5065c2ea089887da434f380c6477797c
-  data.tar.gz: 05dd0a148aed3f8020eb839fd5042298464296ed
+  metadata.gz: 1cfb579022231c0bf26eb16a1156a7e9e5aa24b8
+  data.tar.gz: 6b096b93cce0ddf6995e92058e44a46c0ac59b6f
 SHA512:
-  metadata.gz: 4bffef34b921fe845581381aa759fdeea68bf40ea7b95031f7e8909bd5516828f9b31964b550048bd1507e2dd337394e3c2f51ca38afd814c37c17c610353d40
-  data.tar.gz: 3471c464f75070f30e4ce2bd97e6bb5a10662e779d7cce00b4d0f41f2efdf6d5ba60c77f36a73ad46a81aa48f17ea10ff0e3f6527f9e1f82b7872ebb47fb1fbf
+  metadata.gz: 9f59e117075d751a7a734b8e4ad15e9ef1e1a0bce142285c4f6e7b534d235009ee207d0918219ec5a3fbfdcb928280abc893957587ff5d8059d519cf0f016469
+  data.tar.gz: e171adea68763ffd821fc34d7ef27298122c2155e8238d461aa563e3cf3395a97c8b18ae5a125e9368222d3c9fd08ac9f1e63bea67063ad6d49f194f00065e9e

data/README.md

@@ -34,6 +34,9 @@ Of course, you can use predefined format 'apache' and 'syslog':
       key_name message
     </match>
 
+`fluent-plugin-parser` uses parser plugins of Fluentd (and your own customized parser plugin).
+See document page for more details: http://docs.fluentd.org/articles/parser-plugin-overview
+
 If you want original attribute-data pair in re-emitted message, specify 'reserve_data':
 
     <match raw.apache.*>
@@ -44,30 +47,6 @@ If you want original attribute-data pair in re-emitted message, specify 'reserve
       reserve_data yes
     </match>
 
-Format 'json', 'csv' and 'tsv' is also supported:
-
-    <match raw.sales.*>
-      type parser
-      tag sales
-      format json
-      key_name sales
-    </match>
-
-Format 'ltsv'(Labeled-TSV (Tab separated values)) is also supported:
-
-    <match raw.sales.*>
-      type parser
-      tag sales
-      format ltsv
-      key_name sales
-    </match>
-
-'LTSV' is format like below, unlinke json, easy to write with simple formatter (ex: LogFormat of apache):
-
-    KEY1:VALUE1 [TAB] KEY2:VALUE2 [TAB] ...
-
-About LTSV, see: http://ltsv.org/
-
 If you want to suppress 'pattern not match' log, specify 'suppress\_parse\_error\_log true' to configuration.
 default value is false.
 

data/fluent-plugin-parser.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-parser"
-  gem.version       = "0.3.4"
+  gem.version       = "0.4.0"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to parse single field, or to combine log structure into single field}
@@ -15,5 +15,5 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_development_dependency "rake"
-  gem.add_runtime_dependency "fluentd"
+  gem.add_runtime_dependency "fluentd", ">= 0.10.54"
 end

data/lib/fluent/plugin/out_parser.rb

@@ -1,4 +1,4 @@
-require_relative './fixed_parser'
+require 'fluent/parser'
 
 class Fluent::ParserOutput < Fluent::Output
   Fluent::Plugin.register_output('parser', self)
@@ -11,6 +11,8 @@ class Fluent::ParserOutput < Fluent::Output
   config_param :inject_key_prefix, :string, :default => nil
   config_param :replace_invalid_sequence, :bool, :default => false
   config_param :hash_value_field, :string, :default => nil
+  config_param :suppress_parse_error_log, :bool, :default => false
+  config_param :time_parse, :bool, :default => true
 
   attr_reader :parser
 
@@ -19,11 +21,6 @@ class Fluent::ParserOutput < Fluent::Output
     require 'time'
   end
 
-  # Define `log` method for v0.10.42 or earlier
-  unless method_defined?(:log)
-    define_method("log") { $log }
-  end
-
   def configure(conf)
     super
 
@@ -41,8 +38,15 @@ class Fluent::ParserOutput < Fluent::Output
       @added_prefix_string = @add_prefix + '.'
     end
 
-    @parser = FluentExt::TextParser.new(log())
+    @parser = Fluent::TextParser.new
+    @parser.estimate_current_event = false
     @parser.configure(conf)
+    if !@time_parse && @parser.parser.respond_to?("time_key=".to_sym)
+      # disable parse time
+      @parser.parser.time_key = nil
+    end
+
+    self
   end
 
   def emit(tag, es, chain)
@@ -64,18 +68,28 @@ class Fluent::ParserOutput < Fluent::Output
           end
     es.each do |time,record|
       raw_value = record[@key_name]
-      t,values = raw_value ? parse(raw_value) : [nil, nil]
-      t ||= time
-
-      if values && @inject_key_prefix
-        values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
-      end
-      r = @hash_value_field ? {@hash_value_field => values} : values
-      if @reserve_data
-        r = r ? record.merge(r) : record
-      end
-      if r
-        Fluent::Engine.emit(tag, t, r)
+      begin
+        @parser.parse(raw_value) do |t,values|
+          t ||= time
+          handle_parsed(tag, record, t, values)
+        end
+      rescue Fluent::TextParser::ParserError => e
+        log.warn e.message unless @suppress_parse_error_log
+      rescue ArgumentError => e
+        if @replace_invalid_sequence
+          unless e.message.index("invalid byte sequence in") == 0
+            raise
+          end
+          replaced_string = replace_invalid_byte(raw_value)
+          @parser.parse(replaced_string) do |t,values|
+            t ||= time
+            handle_parsed(tag, record, t, values)
+          end
+        else
+          raise
+        end
+      rescue => e
+        log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
       end
     end
 
@@ -84,17 +98,18 @@ class Fluent::ParserOutput < Fluent::Output
 
   private
 
-  def parse(string)
-    return @parser.parse(string) unless @replace_invalid_sequence
-
-    begin
-      @parser.parse(string)
-    rescue ArgumentError => e
-      unless e.message.index("invalid byte sequence in") == 0
-        raise
-      end
-      replaced_string = replace_invalid_byte(string)
-      @parser.parse(replaced_string)
+  def handle_parsed(tag, record, t, values)
+    if values && @inject_key_prefix
+      values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
+    end
+    r = @hash_value_field ? {@hash_value_field => values} : values
+    if @reserve_data
+      r = r ? record.merge(r) : record
+    end
+    if r
+      Fluent::Engine.emit(tag, t, r)
+    else
+      log.warn "pattern not match #{raw_value}" unless @suppress_parse_error_log
     end
   end
 

data/test/custom_parser.rb

@@ -0,0 +1,39 @@
+module Fluent
+  class TextParser
+    class KVPairParser
+      # key<delim1>value is pair and <pair><delim2><pair> ...
+      # newline splits records
+      include Configurable
+
+      config_param :delim1, :string
+      config_param :delim2, :string
+
+      config_param :time_key, :string, :default => "time"
+      config_param :time_format, :string, :default => nil # time_format is configurable
+
+      def configure(conf)
+        super
+        @time_parser = TimeParser.new(@time_format)
+      end
+
+      def call(text)
+        text.split("\n").each do |line|
+          pairs = text.split(@delim2)
+          record = {}
+          time = nil
+          pairs.each do |pair|
+            k, v = pair.split(@delim1, 2)
+            if k == @time_key
+              time = @time_parser.parse(v)
+            else
+              record[k] = v
+            end
+          end
+          yield time, record
+        end
+      end
+    end
+
+    register_template("kv_pair", Proc.new { KVPairParser.new })
+  end
+end

data/test/plugin/test_out_parser.rb

@@ -304,6 +304,26 @@ class ParserOutputTest < Test::Unit::TestCase
     assert_equal "xxx:first\tyyy:second", first[2]['data']
     assert_equal 'first', second[2]['xxx']
     assert_equal 'second2', second[2]['yyy']
+
+    # convert types
+    d = create_driver(CONFIG_LTSV + %[
+      types i:integer,s:string,f:float,b:bool
+    ], 'foo.baz.test')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.emit({'data' => "i:1\ts:2\tf:3\tb:true\tx:123"}, time)
+    end
+    emits = d.emits
+    assert_equal 1, emits.length
+
+    first = emits[0]
+    assert_equal 'foo.bar.test', first[0]
+    assert_equal time, first[1]
+    assert_equal 1, first[2]['i']
+    assert_equal '2', first[2]['s']
+    assert_equal 3.0, first[2]['f']
+    assert_equal true, first[2]['b']
+    assert_equal '123', first[2]['x']
   end
 
   CONFIG_TSV =  %[
@@ -473,8 +493,6 @@ class ParserOutputTest < Test::Unit::TestCase
     t = Time.now.to_i
     d = create_driver(CONFIG_DONT_PARSE_TIME, 'test.in')
 
-    assert_equal false, d.instance.instance_eval{ @parser }.instance_eval{ @parser }.time_parse
-
     d.run do
       d.emit({'data' => '{"time":1383190430, "f1":"v1"}'}, t)
       d.emit({'data' => '{"time":"1383190430", "f1":"v1"}'}, t)
@@ -515,21 +533,14 @@ class ParserOutputTest < Test::Unit::TestCase
       end
     }
     emits = d.emits
-    assert_equal 2, emits.length
+    assert_equal 1, emits.length
 
     assert_equal 'in', emits[0][0]
-    assert_equal t, emits[0][1]
+    assert_equal 0, emits[0][1]
     assert_equal 'v1', emits[0][2]['f1']
-    assert_equal [], emits[0][2]['time']
-
-    assert_equal 'in', emits[1][0]
-    assert_equal t, emits[1][1]
-    assert_equal 'v1', emits[1][2]['f1']
-    assert_equal 'thisisnottime', emits[1][2]['time']
+    assert_equal 0, emits[0][2]['time'].to_i
   end
 
-
-  #TODO: apache2
   # REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
 
   CONFIG_NOT_REPLACE = %[
@@ -619,26 +630,27 @@ class ParserOutputTest < Test::Unit::TestCase
 
   def swap_logger(instance)
     raise "use with block" unless block_given?
-    parser_logger = instance.parser.log
     dummy = DummyLogger.new
-    instance.parser.log = dummy
-    instance.parser.parser.log = dummy
-
-    restore = if instance.respond_to?("log=".to_sym)
-                saved_logger = instance.log
-                instance.log = dummy
-                lambda{ instance.log = saved_logger; instance.parser.log = instance.parser.parser.log = parser_logger }
-              else
-                saved_logger = $log
-                $log = dummy
-                lambda{ $log = saved_logger; instance.parser.log = instance.parser.parser.log = parser_logger }
-              end
+    saved_logger = instance.log
+    instance.log = dummy
+    restore = lambda{ instance.log = saved_logger }
 
     yield
 
     restore.call
   end
 
+  def test_parser_error_warning
+    d = create_driver(CONFIG_INVALID_TIME_VALUE, 'test.in')
+    swap_logger(d.instance) do
+      assert_raise(DummyLoggerWarnedException) {
+        d.run do
+          d.emit({'data' => '{"time":[], "f1":"v1"}'}, Time.now.to_i)
+        end
+      }
+    end
+  end
+
   def test_suppress_parse_error_log
     # default(disabled) 'suppress_parse_error_log' is not specify
     d = create_driver(CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG, 'test.in')

data/test/plugin/test_out_parser_for_parsers.rb

@@ -0,0 +1,285 @@
+require 'helper'
+require_relative '../custom_parser'
+
+class ParserOutputParsersTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf, tag)
+    Fluent::Test::OutputTestDriver.new(Fluent::ParserOutput, tag).configure(conf)
+  end
+
+  def test_regexp_parser
+    # exists in test_out_parser
+  end
+
+  def test_json_parser
+    # exists in test_out_parser
+  end
+
+  def test_tsv_parser
+    # exists in test_out_parser
+  end
+
+  def test_ltsv_parser
+    # exists in test_out_parser
+  end
+
+  def test_csv_parser
+    # exists in test_out_parser
+  end
+
+  def test_none_parser
+    d = create_driver(<<EOF, 'test.in')
+remove_prefix test
+add_prefix    parsed
+key_name      message
+format        none
+EOF
+    time = Time.parse("2014-11-05 15:59:30").to_i
+    d.run do
+      d.emit({"message" => "aaaa bbbb cccc 1"}, time)
+      d.emit({"message" => "aaaa bbbb cccc 2"}, time)
+      d.emit({"message" => "aaaa bbbb cccc 3"}, time)
+      d.emit({"message" => "aaaa bbbb cccc 4"}, time)
+    end
+
+    e = d.emits
+    assert_equal 4, e.length
+
+    assert_equal 'parsed.in', e[0][0]
+    assert_equal time, e[0][1]
+    assert_equal 'aaaa bbbb cccc 1', e[0][2]['message']
+
+    assert_equal 'parsed.in', e[1][0]
+    assert_equal time, e[1][1]
+    assert_equal 'aaaa bbbb cccc 2', e[1][2]['message']
+
+    assert_equal 'parsed.in', e[2][0]
+    assert_equal time, e[2][1]
+    assert_equal 'aaaa bbbb cccc 3', e[2][2]['message']
+
+    assert_equal 'parsed.in', e[3][0]
+    assert_equal time, e[3][1]
+    assert_equal 'aaaa bbbb cccc 4', e[3][2]['message']
+  end
+
+  def test_apache_parser
+    log1 = '127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326'
+    log2 = '127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"'
+    log_time = Time.parse("2000-10-10 13:55:36 -0700").to_i
+
+    d = create_driver(<<EOF, 'test.in')
+remove_prefix test
+add_prefix    parsed
+key_name      message
+format        apache
+EOF
+    time = Time.parse("2014-11-05 15:59:30").to_i
+    d.run do
+      d.emit({"message" => log1}, time)
+      d.emit({"message" => log2}, time)
+    end
+
+    e = d.emits
+    assert_equal 2, e.length
+
+    assert_equal 'parsed.in', e[0][0]
+    assert_equal log_time, e[0][1]
+    assert_equal '127.0.0.1', e[0][2]['host']
+    assert_equal 'frank', e[0][2]['user']
+    assert_equal 'GET', e[0][2]['method']
+    assert_equal '/apache_pb.gif', e[0][2]['path']
+    assert_equal '200', e[0][2]['code']
+    assert_equal '2326', e[0][2]['size']
+    assert_nil e[0][2]['referer']
+    assert_nil e[0][2]['agent']
+
+    assert_equal 'parsed.in', e[1][0]
+    assert_equal log_time, e[1][1]
+    assert_equal '127.0.0.1', e[1][2]['host']
+    assert_equal 'frank', e[1][2]['user']
+    assert_equal 'GET', e[1][2]['method']
+    assert_equal '/apache_pb.gif', e[1][2]['path']
+    assert_equal '200', e[1][2]['code']
+    assert_equal '2326', e[1][2]['size']
+    assert_equal 'http://www.example.com/start.html', e[1][2]['referer']
+    assert_equal 'Mozilla/4.08 [en] (Win98; I ;Nav)', e[1][2]['agent']
+  end
+
+  def test_apache_parser_with_types
+    log = '127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"'
+    log_time = Time.parse("2000-10-10 13:55:36 -0700").to_i
+
+    d = create_driver(<<EOF, 'test.in')
+remove_prefix test
+add_prefix    parsed
+key_name      message
+format        apache
+types code:integer,size:integer
+EOF
+    time = Time.parse("2014-11-05 15:59:30").to_i
+    d.run do
+      d.emit({"message" => log}, time)
+    end
+
+    e = d.emits
+    assert_equal 1, e.length
+
+    assert_equal 'parsed.in', e[0][0]
+    assert_equal log_time, e[0][1]
+    assert_equal '127.0.0.1', e[0][2]['host']
+    assert_equal 'frank', e[0][2]['user']
+    assert_equal 'GET', e[0][2]['method']
+    assert_equal '/apache_pb.gif', e[0][2]['path']
+    assert_equal 200, e[0][2]['code']
+    assert_equal 2326, e[0][2]['size']
+    assert_equal 'http://www.example.com/start.html', e[0][2]['referer']
+    assert_equal 'Mozilla/4.08 [en] (Win98; I ;Nav)', e[0][2]['agent']
+  end
+
+  def test_syslog_parser
+    loglines = <<LOGS
+Nov  5 16:19:48 myhost.local netbiosd[50]: name servers down?
+Nov  5 16:21:20 myhost.local coreaudiod[320]: Disabled automatic stack shots because audio IO is active
+Nov  5 16:21:20 myhost.local coreaudiod[320]: Enabled automatic stack shots because audio IO is inactive
+LOGS
+    logs = loglines.split("\n").reject(&:empty?)
+
+    d = create_driver(<<EOF, 'test.in')
+remove_prefix test
+add_prefix    parsed
+key_name      message
+format        syslog
+EOF
+    time = Time.parse("2014-11-05 15:59:30").to_i
+    d.run do
+      d.emit({"message" => logs[0]}, time)
+      d.emit({"message" => logs[1]}, time)
+      d.emit({"message" => logs[2]}, time)
+    end
+
+    emits = d.emits
+    assert_equal 3, emits.length
+
+    e = emits[0]
+    assert_equal 'parsed.in', e[0]
+    assert_equal Time.parse("2014-11-05 16:19:48").to_i, e[1]
+    r = e[2]
+    assert_equal 'myhost.local', r['host']
+    assert_equal 'netbiosd', r['ident']
+    assert_equal '50', r['pid']
+    assert_equal 'name servers down?', r['message']
+
+    e = emits[1]
+    assert_equal 'parsed.in', e[0]
+    assert_equal Time.parse("2014-11-05 16:21:20").to_i, e[1]
+    r = e[2]
+    assert_equal 'myhost.local', r['host']
+    assert_equal 'coreaudiod', r['ident']
+    assert_equal '320', r['pid']
+    assert_equal 'Disabled automatic stack shots because audio IO is active', r['message']
+
+    e = emits[2]
+    assert_equal 'parsed.in', e[0]
+    assert_equal Time.parse("2014-11-05 16:21:20").to_i, e[1]
+    r = e[2]
+    assert_equal 'myhost.local', r['host']
+    assert_equal 'coreaudiod', r['ident']
+    assert_equal '320', r['pid']
+    assert_equal 'Enabled automatic stack shots because audio IO is inactive', r['message']
+  end
+
+  def x_test_multiline_parser
+    # I can't configure this format well...
+    log1 = <<LOG
+*** 2014/11/05 16:33:01 -0700
+  host: myhost
+  port: 2048
+  message: first line
+LOG
+    log2 = <<LOG
+*** 2014/11/05 16:33:02 +0900
+  host: myhost
+  port: 2049
+  message: second line
+LOG
+    log3 = <<LOG
+*** 2014/11/05 16:43:11 +1100
+LOG
+    d = create_driver(<<'EOF', 'test.in')
+remove_prefix test
+add_prefix    parsed
+key_name      message
+format        multiline
+time_format %Y/%m/%d %H:%M:%S %z
+format_firstline /^\*\*\* /
+format1 /\*\*\* (?<time>\d{4}/\d\d/\d\d/ \d\d:\d\d:\d\d [-+]\d{4})/
+format2 /\s*host: (?<host>[^\s]+)/
+format3 /\s*port: (?<port>\d+)/
+format4 /\s*message: (?<message>[^ ]*)/
+EOF
+    time = Time.parse("2014-11-05 15:59:30").to_i
+    d.run do
+      d.emit({"message" => log1}, time)
+      d.emit({"message" => log2}, time)
+      d.emit({"message" => log3}, time)
+    end
+
+    emits = d.emits
+    assert_equal 2, emits.length
+
+    e = emits[0]
+    assert_equal 'parsed.in', e[0]
+    assert_equal Time.parse("2014-11-05 16:33:01 -0700").to_i, e[1]
+    r = e[2]
+    assert_equal 'myhost', r['host']
+    assert_equal '2048', r['port']
+    assert_equal 'first line', r['message']
+
+    e = emits[1]
+    assert_equal 'parsed.in', e[0]
+    assert_equal Time.parse("2014-11-05 16:33:02 +0900").to_i, e[1]
+    r = e[2]
+    assert_equal 'myhost', r['host']
+    assert_equal '2049', r['port']
+    assert_equal 'second line', r['message']
+  end
+
+  def test_custom_parser
+    d = create_driver(<<'EOF', 'test.in')
+remove_prefix test
+add_prefix    parsed
+key_name      message
+format        kv_pair
+time_format %Y-%m-%d %H:%M:%S %z
+delim1 :
+delim2 ,
+EOF
+    time = Time.parse("2014-11-05 15:59:30").to_i
+    d.run do
+      d.emit({"message" => "k1:v1,k2:v2,k3:1,time:2014-11-05 00:00:00 +0000"}, time)
+      d.emit({"message" => "k1:v1,k2:v2,k3:2"}, time) # original time is used
+      d.emit({"message" => "k1:v1,k2:v2,k3:3,time:2014-11-05 00:00:00"}, time) # time parse error -> not emitted
+    end
+    emits = d.emits
+    assert_equal 2, emits.length
+
+    e = emits[0]
+    assert_equal 'parsed.in', e[0]
+    assert_equal Time.parse("2014-11-05 00:00:00 +0000").to_i, e[1]
+    r = e[2]
+    assert_equal 'v1', r['k1']
+    assert_equal 'v2', r['k2']
+    assert_equal '1', r['k3']
+
+    e = emits[1]
+    assert_equal 'parsed.in', e[0]
+    assert_equal Time.parse("2014-11-05 15:59:30").to_i, e[1]
+    r = e[2]
+    assert_equal 'v1', r['k1']
+    assert_equal 'v2', r['k2']
+    assert_equal '2', r['k3']
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors:
 - TAGOMORI Satoshi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-21 00:00:00.000000000 Z
+date: 2014-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.54
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.54
 description: fluentd plugin to parse single field, or to combine log structure into
   single field
 email:
@@ -53,12 +53,13 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-parser.gemspec
-- lib/fluent/plugin/fixed_parser.rb
 - lib/fluent/plugin/out_deparser.rb
 - lib/fluent/plugin/out_parser.rb
+- test/custom_parser.rb
 - test/helper.rb
 - test/plugin/test_deparser.rb
 - test/plugin/test_out_parser.rb
+- test/plugin/test_out_parser_for_parsers.rb
 homepage: https://github.com/tagomoris/fluent-plugin-parser
 licenses:
 - APLv2
@@ -84,6 +85,8 @@ signing_key:
 specification_version: 4
 summary: plugin to parse/combine fluentd log messages
 test_files:
+- test/custom_parser.rb
 - test/helper.rb
 - test/plugin/test_deparser.rb
 - test/plugin/test_out_parser.rb
+- test/plugin/test_out_parser_for_parsers.rb

data/lib/fluent/plugin/fixed_parser.rb

@@ -1,280 +0,0 @@
-#
-# This module is copied from fluentd/lib/fluent/parser.rb and
-# fixed not to overwrite 'time' (reserve nil) when time not found in parsed string.
-module FluentExt; end
-
-class FluentExt::TextParser
-  class GenericParser
-    include Fluent::Configurable
-
-    config_param :time_key, :string, :default => 'time'
-    config_param :time_format, :string, :default => nil
-    config_param :time_parse, :bool, :default => true
-
-    attr_accessor :log
-
-    def initialize
-      super
-
-      @cache1_key = nil
-      @cache1_time = nil
-      @cache2_key = nil
-      @cache2_time = nil
-
-      @log = nil
-    end
-
-    def parse_time(record)
-      time = nil
-
-      unless @time_parse
-        return time, record
-      end
-
-      if value = record.delete(@time_key)
-        if @cache1_key == value
-          time = @cache1_time
-        elsif @cache2_key == value
-          time = @cache2_time
-        else
-          begin
-            time = if @time_format
-                     Time.strptime(value, @time_format).to_i
-                   else
-                     Time.parse(value).to_i
-                   end
-            @cache1_key = @cache2_key
-            @cache1_time = @cache2_time
-            @cache2_key = value
-            @cache2_time = time
-          rescue TypeError, ArgumentError => e
-            @log.warn "Failed to parse time", :key => @time_key, :value => value
-            record[@time_key] = value
-          end
-        end
-      end
-
-      return time, record
-    end
-  end
-
-  class RegexpParser < GenericParser
-    include Fluent::Configurable
-
-    config_param :suppress_parse_error_log, :bool, :default => false
-
-    def initialize(regexp, conf={})
-      super()
-      @regexp = regexp
-      unless conf.empty?
-        configure(conf)
-      end
-    end
-
-    def call(text)
-      m = @regexp.match(text)
-      unless m
-        unless @suppress_parse_error_log
-          @log.warn "pattern not match: #{text}"
-        end
-
-        return nil, nil
-      end
-
-      record = {}
-      m.names.each {|name|
-        record[name] = m[name] if m[name]
-      }
-      parse_time(record)
-    end
-  end
-
-  class JSONParser < GenericParser
-    def call(text)
-      record = Yajl.load(text)
-      return parse_time(record)
-    rescue Yajl::ParseError
-      unless @suppress_parse_error_log
-        @log.warn "pattern not match(json): #{text.inspect}: #{$!}"
-      end
-
-      return nil, nil
-    end
-  end
-
-  class LabeledTSVParser < GenericParser
-    def call(text)
-      record = Hash[text.split("\t").map{|p| p.split(":", 2)}]
-      parse_time(record)
-    end
-  end
-
-  class ValuesParser < GenericParser
-    config_param :keys, :string
-
-    def configure(conf)
-      super
-      @keys = @keys.split(",")
-    end
-
-    def values_map(values)
-      Hash[@keys.zip(values)]
-    end
-  end
-
-  class TSVParser < ValuesParser
-    config_param :delimiter, :string, :default => "\t"
-
-    def call(text)
-      return parse_time(values_map(text.split(@delimiter)))
-    end
-  end
-
-  class CSVParser < ValuesParser
-    def initialize
-      super
-      require 'csv'
-    end
-
-    def call(text)
-      return parse_time(values_map(CSV.parse_line(text)))
-    end
-  end
-
-  class ApacheParser < GenericParser
-    include Fluent::Configurable
-
-    REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
-
-    def initialize
-      super
-
-      @time_key = "time"
-      @time_format = "%d/%b/%Y:%H:%M:%S %z"
-   end
-
-    def call(text)
-      m = REGEXP.match(text)
-      unless m
-        unless @suppress_parse_error_log
-          @log.warn "pattern not match: #{text.inspect}"
-        end
-
-        return nil, nil
-      end
-
-      host = m['host']
-      host = (host == '-') ? nil : host
-
-      user = m['user']
-      user = (user == '-') ? nil : user
-
-      time = m['time']
-
-      method = m['method']
-      path = m['path']
-
-      code = m['code'].to_i
-      code = nil if code == 0
-
-      size = m['size']
-      size = (size == '-') ? nil : size.to_i
-
-      referer = m['referer']
-      referer = (referer == '-') ? nil : referer
-
-      agent = m['agent']
-      agent = (agent == '-') ? nil : agent
-
-      record = {
-        "time" => time,
-        "host" => host,
-        "user" => user,
-        "method" => method,
-        "path" => path,
-        "code" => code,
-        "size" => size,
-        "referer" => referer,
-        "agent" => agent,
-      }
-
-      parse_time(record)
-    end
-  end
-
-  TEMPLATE_FACTORIES = {
-    'apache' => Proc.new { RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
-    'apache2' => Proc.new { ApacheParser.new },
-    'nginx' => Proc.new { RegexpParser.new(/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,  {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
-    'syslog' => Proc.new { RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}) },
-    'json' => Proc.new { JSONParser.new },
-    'csv' => Proc.new { CSVParser.new },
-    'tsv' => Proc.new { TSVParser.new },
-    'ltsv' => Proc.new { LabeledTSVParser.new },
-  }
-
-  def self.register_template(name, regexp_or_proc, time_format=nil)
-
-    factory = if regexp_or_proc.is_a?(Regexp)
-                regexp = regexp_or_proc
-                Proc.new { RegexpParser.new(regexp, {'time_format'=>time_format}) }
-              else
-                Proc.new { proc }
-              end
-    TEMPLATE_FACTORIES[name] = factory
-  end
-
-  attr_accessor :log
-  attr_reader :parser
-
-  def initialize(logger)
-    @log = logger
-    @parser = nil
-  end
-
-  def configure(conf, required=true)
-    format = conf['format']
-
-    if format == nil
-      if required
-        raise Fluent::ConfigError, "'format' parameter is required"
-      else
-        return nil
-      end
-    end
-
-    if format[0] == ?/ && format[format.length-1] == ?/
-      # regexp
-      begin
-        regexp = Regexp.new(format[1..-2])
-        if regexp.named_captures.empty?
-          raise "No named captures"
-        end
-      rescue
-        raise Fluent::ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
-      end
-      @parser = RegexpParser.new(regexp)
-
-    else
-      # built-in template
-      factory = TEMPLATE_FACTORIES[format]
-      unless factory
-        raise Fluent::ConfigError, "Unknown format template '#{format}'"
-      end
-      @parser = factory.call
-
-    end
-
-    @parser.log = @log
-
-    if @parser.respond_to?(:configure)
-      @parser.configure(conf)
-    end
-
-    return true
-  end
-
-  def parse(text)
-    return @parser.call(text)
-  end
-end