fluent-plugin-parser 0.2.0 → 0.2.1

data/README.md

@@ -43,7 +43,7 @@ If you want original attribute-data pair in re-emitted message, specify 'reserve
       reserve_data yes
     </match>
 
-Format 'json' is also supported:
+Format 'json', 'csv' and 'tsv' is also supported:
 
     <match raw.sales.*>
       type parser

data/fluent-plugin-parser.gemspec

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-parser"
-  gem.version       = "0.2.0"
+  gem.version       = "0.2.1"
   gem.authors       = ["TAGOMORI Satoshi"]
   gem.email         = ["tagomoris@gmail.com"]
   gem.description   = %q{fluentd plugin to parse single field, or to combine log structure into single field}

data/lib/fluent/plugin/fixed_parser.rb

@@ -20,8 +20,7 @@ class FluentExt::TextParser
     def call(text)
       m = @regexp.match(text)
       unless m
-        $log.debug "pattern not match: #{text}"
-        # TODO?
+        $log.warn "pattern not match: #{text}"
         return nil, nil
       end
 
@@ -55,15 +54,13 @@ class FluentExt::TextParser
 
     def parse_time(record)
       time = nil
-      
       if value = record.delete(@time_key)
-        if @time_format
-          time = Time.strptime(value, @time_format).to_i
-        else
-          time = value.to_i
-        end
+        time = if @time_format
+                 Time.strptime(value, @time_format).to_i
+               else
+                 Time.parse(value).to_i
+               end
       end
-
       return time, record
     end
   end
@@ -73,6 +70,7 @@ class FluentExt::TextParser
       record = Yajl.load(text)
       return parse_time(record)
     rescue Yajl::ParseError
+      $log.warn "pattern not match(json): #{text.inspect}: #{$!}"
       return nil, nil
     end
   end
@@ -84,22 +82,109 @@ class FluentExt::TextParser
     end
   end
 
-  TEMPLATES = {
-    'apache' => RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}),
-    'syslog' => RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}),
-    'json' => JSONParser.new,
-    'ltsv' => LabeledTSVParser.new,
-  }
+  class ValuesParser < GenericParser
+    config_param :keys, :string
 
-  def self.register_template(name, regexp_or_proc, time_format=nil)
-    if regexp_or_proc.is_a?(Regexp)
-      pr = regexp_or_proc
-    else
-      regexp = regexp_or_proc
-      pr = RegexpParser.new(regexp, {'time_format'=>time_format})
+    def configure(conf)
+      super
+      @keys = @keys.split(",")
+    end
+
+    def values_map(values)
+      Hash[@keys.zip(values)]
+    end
+  end
+
+  class TSVParser < ValuesParser
+    config_param :delimiter, :string, :default => "\t"
+
+    def call(text)
+      return parse_time(values_map(text.split(@delimiter)))
+    end
+  end
+
+  class CSVParser < ValuesParser
+    def initialize
+      super
+      require 'csv'
+    end
+
+    def call(text)
+      return parse_time(values_map(CSV.parse_line(text)))
+    end
+  end
+
+  class ApacheParser
+    include Fluent::Configurable
+
+    REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+
+    def call(text)
+      m = REGEXP.match(text)
+      unless m
+        $log.warn "pattern not match: #{text.inspect}"
+        return nil, nil
+      end
+
+      host = m['host']
+      host = (host == '-') ? nil : host
+
+      user = m['user']
+      user = (user == '-') ? nil : user
+      
+      time = m['time']
+      time = Time.strptime(time, "%d/%b/%Y:%H:%M:%S %z").to_i
+      
+      method = m['method']
+      path = m['path']
+      
+      code = m['code'].to_i 
+      code = nil if code == 0
+
+      size = m['size']
+      size = (size == '-') ? nil : size.to_i
+      
+      referer = m['referer']
+      referer = (referer == '-') ? nil : referer
+      
+      agent = m['agent']
+      agent = (agent == '-') ? nil : agent
+      
+      record = {
+        "host" => host,
+        "user" => user,
+        "method" => method,
+        "path" => path,
+        "code" => code,
+        "size" => size,
+        "referer" => referer,
+        "agent" => agent,
+      } 
+
+      return time, record
     end
+  end
 
-    TEMPLATES[name] = pr
+  TEMPLATE_FACTORIES = {
+    'apache' => Proc.new { RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
+    'apache2' => Proc.new { ApacheParser.new },
+    'nginx' => Proc.new { RegexpParser.new(/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,  {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
+    'syslog' => Proc.new { RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}) },
+    'json' => Proc.new { JSONParser.new },
+    'csv' => Proc.new { CSVParser.new },
+    'tsv' => Proc.new { TSVParser.new },
+    'ltsv' => Proc.new { LabeledTSVParser.new },
+  }
+
+  def self.register_template(name, regexp_or_proc, time_format=nil)
+    
+    factory = if regexp_or_proc.is_a?(Regexp)
+                regexp = regexp_or_proc
+                Proc.new { RegexpParser.new(regexp, {'time_format'=>time_format}) }
+              else
+                Proc.new { proc }
+              end
+    TEMPLATE_FACTORIES[name] = factory
   end
 
   def initialize
@@ -127,15 +212,16 @@ class FluentExt::TextParser
       rescue
         raise Fluent::ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
       end
-
       @parser = RegexpParser.new(regexp)
 
     else
       # built-in template
-      @parser = TEMPLATES[format]
-      unless @parser
+      factory = TEMPLATE_FACTORIES[format]
+      unless factory
         raise ConfigError, "Unknown format template '#{format}'"
       end
+      @parser = factory.call
+
     end
 
     if @parser.respond_to?(:configure)

data/test/plugin/test_out_parser.rb

@@ -261,4 +261,58 @@ class ParserOutputTest < Test::Unit::TestCase
     assert_equal 'first', second[2]['xxx']
     assert_equal 'second2', second[2]['yyy']
   end
+
+  CONFIG_TSV =  %[
+    remove_prefix foo.baz
+    add_prefix foo.bar
+    format tsv
+    key_name data
+    keys key1,key2,key3
+  ]
+  def test_emit_ltsv
+    d = create_driver(CONFIG_TSV, 'foo.baz.test')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.emit({'data' => "value1\tvalue2\tvalueThree", 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    emits = d.emits
+    assert_equal 1, emits.length
+
+    first = emits[0]
+    assert_equal 'foo.bar.test', first[0]
+    assert_equal time, first[1]
+    assert_nil first[2]['data']
+    assert_equal 'value1', first[2]['key1']
+    assert_equal 'value2', first[2]['key2']
+    assert_equal 'valueThree', first[2]['key3']
+  end
+
+  CONFIG_CSV =  %[
+    remove_prefix foo.baz
+    add_prefix foo.bar
+    format csv
+    key_name data
+    keys key1,key2,key3
+  ]
+  def test_emit_ltsv
+    d = create_driver(CONFIG_CSV, 'foo.baz.test')
+    time = Time.parse("2012-04-02 18:20:59").to_i
+    d.run do
+      d.emit({'data' => 'value1,"value2","value""ThreeYes!"', 'xxx' => 'x', 'yyy' => 'y'}, time)
+    end
+    emits = d.emits
+    assert_equal 1, emits.length
+
+    first = emits[0]
+    assert_equal 'foo.bar.test', first[0]
+    assert_equal time, first[1]
+    assert_nil first[2]['data']
+    assert_equal 'value1', first[2]['key1']
+    assert_equal 'value2', first[2]['key2']
+    assert_equal 'value"ThreeYes!', first[2]['key3']
+  end
+
+  #TODO: apache2
+  # REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-parser
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-06 00:00:00.000000000 Z
+date: 2013-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake