fluent-plugin-grok-parser 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cb4cafd272f76ee654cf6d3b71dcf8f9090d58f1
-  data.tar.gz: 4b314f74bac26038137a979ce8396bc299f745f1
+  metadata.gz: a517f65e8155156ff9151628ddead36b679d3105
+  data.tar.gz: 7a034a4e0b8d28234ba30612bcf7fd205aa3fbb3
 SHA512:
-  metadata.gz: 97235a3241c2b2f536fed27660a2e13ae6f9f01a46fbf05c5cc5db87093d0e60af3ccbef2d73b51ffb007099d9f8299530d45633ac4103bbeb225d268c5cf475
-  data.tar.gz: 86b2052f11b2a00a0a63bdbc53aa43b0d41d16fde1e30fa1424f4b120c5af79e8b601a40aafe2b5824ffb41d89fce30ba014d5ff10b4026a5560719659c6080e
+  metadata.gz: a16eb5eff0997a710e4dcb7d92ec60ebea27d151d8a85b61c7b7230f4579aeabaf3f54c42d0789f564d579c14eb25c4adbb5c9af0e2527336b9d3e6cb030527b
+  data.tar.gz: 65b1cef569bcf821cdee87dc322088448cc5dfd6fdff08bb4fe9330867e2094a4e848fe660b41d7302b8cc682c537d7d458286083d3566783d04b152c4db5723

data/Gemfile.lock

@@ -1,25 +1,34 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-grok-parser (0.0.1)
-      fluentd
+    fluent-plugin-grok-parser (0.0.2)
+      fluentd (>= 0.10.58)
 
 GEM
   remote: https://rubygems.org/
   specs:
     cool.io (1.2.4)
-    fluentd (0.10.49)
-      cool.io (>= 1.1.1, < 2.0.0, != 1.2.0)
+    fluentd (0.12.1)
+      cool.io (>= 1.2.2, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       json (>= 1.4.3)
-      msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      msgpack (>= 0.5.4, < 0.6.0)
       sigdump (~> 0.2.2)
+      string-scrub (>= 0.0.3)
+      tzinfo (>= 1.0.0)
+      tzinfo-data (>= 1.0.0)
       yajl-ruby (~> 1.0)
     http_parser.rb (0.6.0)
     json (1.8.1)
-    msgpack (0.5.8)
+    msgpack (0.5.9)
     rake (10.1.1)
     sigdump (0.2.2)
+    string-scrub (0.0.5)
+    thread_safe (0.3.4)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2014.10)
+      tzinfo (>= 1.0.0)
     yajl-ruby (1.2.1)
 
 PLATFORMS

data/LICENSE

@@ -0,0 +1,44 @@
+Copyright 2014 Kiyoto Tamura
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+For Grok's Ruby implementation found in parser_grok.rb
+
+Copyright 2009-2013 Jordan Sissel
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+For Grok patterns (under patterns/):
+
+Copyright 2009-2013 Jordan Sissel, Pete Fritchman, and contributors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -4,7 +4,7 @@ This is a Fluentd plugin to enable Logstash's Grok-like parsing logic.
 
 ## What's Grok?
 
-Grok is a macro to simplify and reuse regexes, originally developed by [Jordan Sissel](http://github.com/semicomplete).
+Grok is a macro to simplify and reuse regexes, originally developed by [Jordan Sissel](http://github.com/jordansissel).
 
 This is a partial implementation of Grok's grammer that should meet most of the needs.
 
@@ -13,12 +13,34 @@ This is a partial implementation of Grok's grammer that should meet most of the
 You can use it wherever you used the `format` parameter to parse texts. In the following example, it
 extracts the first IP address that matches in the log.
 
-```
+```aconf
 <source>
   type tail
   path /path/to/log
   format grok
   grok_pattern %{IP:ip_address}
+  tag grokked_log
+</source>
+```
+
+**If you want to try multiple grok patterns and use the first matched one**, you can use the following syntax:
+
+```aconf
+<source>
+  type tail
+  path /path/to/log
+  format grok
+  <grok>
+    pattern %{COMBINEDAPACHELOG}
+    time_format "%d/%b/%Y:%H:%M:%S %z"
+  </grok>
+  <grok>
+    pattern %{IP:ip_address}
+  </grok>
+  <grok>
+    pattern %{GREEDYDATA:message}
+  </grok>
+  tag grokked_log
 </source>
 ```
 

data/fluent-plugin-grok-parser.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-grok-parser"
-  spec.version       = "0.0.1"
+  spec.version       = "0.0.2"
   spec.authors       = ["kiyoto"]
   spec.email         = ["kiyoto@treasure-data.com"]
   spec.summary       = %q{Fluentd plugin to suppor Logstash-inspired Grok format for parsing logs}
@@ -18,5 +18,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
-  spec.add_runtime_dependency "fluentd"
+  spec.add_runtime_dependency "fluentd", ">=0.10.58"
 end

data/lib/fluent/plugin/parser_grok.rb

@@ -2,12 +2,14 @@ module Fluent
   class TextParser
     class GrokPatternNotFoundError < Exception; end
 
-    class GrokParser
-      include Configurable
+    class GrokParser < Parser
+      Plugin.register_parser('grok', self)
       config_param :time_format, :string, :default => nil
-      config_param :grok_pattern, :string
+      config_param :grok_pattern, :string, :default => nil
       config_param :custom_pattern_path, :string, :default => nil
 
+      # Much of the Grok implementation is based on Jordan Sissel's jls-grok
+      # See https://github.com/jordansissel/ruby-grok/blob/master/lib/grok-pure.rb
       PATTERN_RE = \
           /%\{    # match '%{' not prefixed with '\'
              (?<name>     # match the pattern name
@@ -23,6 +25,8 @@ module Fluent
         Dir.glob(default_pattern_dir) do |pattern_file_path|
           add_patterns_from_file(pattern_file_path)
         end
+        @default_parser = NoneParser.new
+        @parsers = []
       end
 
       def configure(conf={})
@@ -38,12 +42,13 @@ module Fluent
           end
         end
 
-        begin
-          regexp = expand_pattern(conf['grok_pattern'])
-          $log.info "Expanded the pattern #{conf['grok_pattern']} into #{regexp}"
-          @parser = RegexpParser.new(Regexp.new(regexp), conf)
-        rescue => e
-          $log.error e.backtrace
+        if @grok_pattern
+          @parsers = [expand_pattern_exn(@grok_pattern, conf)]
+        else
+          grok_confs = conf.elements.select {|e| e.name == 'grok'}
+          grok_confs.each do |grok_conf|
+            @parsers << expand_pattern_exn(grok_conf['pattern'], grok_conf)
+          end
         end
       end
 
@@ -55,6 +60,14 @@ module Fluent
         end
       end
 
+      def expand_pattern_exn(pattern, conf)
+        regexp = expand_pattern(pattern)
+        $log.info "Expanded the pattern #{conf['grok_pattern']} into #{regexp}"
+        RegexpParser.new(Regexp.new(regexp), conf)
+      rescue => e
+        $log.error e.backtrace.join("\n")
+      end
+
       def expand_pattern(pattern)
         # It's okay to modify in place. no need to expand it more than once.
         while true
@@ -73,15 +86,28 @@ module Fluent
         pattern 
       end
 
-      def call(text, &block)
-        if block
-          @parser.call(text, &block)
+      def parse(text)
+        if block_given?
+          @parsers.each do |parser|
+            parser.parse(text) do |time, record|
+              if time and record
+                yield time, record
+                return
+              end
+            end
+          end
+          yield @default_parser.parse(text)
         else
-          @parser.call(text)
+          @parsers.each do |parser|
+            parser.parse(text) do |time, record|
+              if time and record
+                return time, record
+              end
+            end
+          end
+          return @default_parser.parse(text)
         end
       end
     end
-
-    TextParser.register_template('grok', Proc.new { GrokParser.new })
   end
 end

data/test/test_grok_parser.rb

@@ -18,7 +18,14 @@ class GrokParserTest < ::Test::Unit::TestCase
   def internal_test_grok_pattern(grok_pattern, text, expected_time, expected_record, options = {})
     parser = TextParser::GrokParser.new
     parser.configure({"grok_pattern" => grok_pattern}.merge(options))
-    [parser.call(text), parser.call(text) { |time, record| return time, record}].each { |time, record|
+
+    # for the old, return based API
+    time, record = parser.parse(text)
+    assert_equal(expected_time, time) if expected_time
+    assert_equal(expected_record, record)
+
+    # for the new API
+    parser.parse(text) {|time, record|
       assert_equal(expected_time, time) if expected_time
       assert_equal(expected_record, record)
     }
@@ -46,7 +53,7 @@ class GrokParserTest < ::Test::Unit::TestCase
                                {"mac_address" => "DEAD.BEEF.1234", "ip_address" => "127.0.0.1"})
   end
 
-  def test_call_for_apache_pattern
+  def test_call_for_complex_pattern
     internal_test_grok_pattern('%{COMBINEDAPACHELOG}', '127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"',
                                 str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'),
                                 {
@@ -65,22 +72,6 @@ class GrokParserTest < ::Test::Unit::TestCase
                               )
   end
 
-  def test_call_for_nagios_pattern
-    internal_test_grok_pattern('%{NAGIOSLOGLINE}', '[1404239939] SERVICE ALERT: servername;PING;OK;SOFT;2;PING OK - Packet loss = 16%, RTA = 72.18 ms',
-                               str2time('1404239939', '%s'),
-                               {
-                                 "nagios_message" => "PING OK - Packet loss = 16%, RTA = 72.18 ms",
-                                 "nagios_type" => "SERVICE ALERT",
-                                 "nagios_hostname" => "servername",
-                                 "nagios_service" => "PING",
-                                 "nagios_state" => "OK",
-                                 "nagios_statelevel" => "SOFT",
-                                 "nagios_attempt" => "2"
-                               },
-                               "time_format" => "%s"
-                              )
-  end
-
   def test_call_for_custom_pattern
     pattern_file = File.new(File.expand_path("../my_pattern", __FILE__), "w")
     pattern_file.write("MY_AWESOME_PATTERN %{GREEDYDATA:message}\n")

data/test/test_grok_parser_in_tcp.rb

@@ -0,0 +1,89 @@
+require 'helper'
+require 'fluent/test'
+
+class TcpInputWithGrokTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  PORT = unused_port
+  BASE_CONFIG = %[
+    port #{PORT}
+    tag tcp
+    format grok
+  ]
+  CONFIG = BASE_CONFIG + %[
+    bind 127.0.0.1
+  ]
+  IPv6_CONFIG = BASE_CONFIG + %[
+    bind ::1
+  ]
+
+  def create_driver(conf)
+    Fluent::Test::InputTestDriver.new(Fluent::TcpInput).configure(conf)
+  end
+
+  def test_configure
+    configs = {'127.0.0.1' => CONFIG}
+    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+
+    configs.each_pair { |k, v|
+      d = create_driver(v)
+      assert_equal PORT, d.instance.port
+      assert_equal k, d.instance.bind
+      assert_equal "\n", d.instance.delimiter
+    }
+  end
+
+  def test_grok_pattern
+    tests = [
+      {'msg' => "tcptest1\n", 'expected' => 'tcptest1'},
+      {'msg' => "tcptest2\n", 'expected' => 'tcptest2'},
+    ]
+    block_config = %[
+      <grok>
+        pattern ${GREEDYDATA:message}
+      </grok>
+    ]
+
+    internal_test_grok('grok_pattern %{GREEDYDATA:message}', tests)
+    internal_test_grok(block_config, tests)
+  end
+
+  def test_grok_multi_patterns
+    tests = [
+      {'msg' => "Current time is 2014-01-01T00:00:00+0900\n", 'expected' => '2014-01-01T00:00:00+0900'},
+      {'msg' => "The first word matches\n", 'expected' => 'The'}
+    ]
+    block_config = %[
+      <grok>
+        pattern %{TIMESTAMP_ISO8601:message}
+      </grok>
+      <grok>
+        pattern %{WORD:message}
+      </grok>
+    ]
+    internal_test_grok(block_config, tests)
+  end
+
+  def internal_test_grok(conf, tests)
+    d = create_driver(BASE_CONFIG + conf)
+    d.run do
+      tests.each {|test|
+        TCPSocket.open('127.0.0.1', PORT) do |s|
+          s.send(test['msg'], 0)
+        end
+      }
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests)
+  end
+
+  def compare_test_result(emits, tests)
+    assert_equal(2, emits.size)
+    emits.each_index {|i|
+      assert_equal(tests[i]['expected'], emits[i][2]['message'])
+    }
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grok-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - kiyoto
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-01 00:00:00.000000000 Z
+date: 2015-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.58
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.10.58
 description: 
 email:
 - kiyoto@treasure-data.com
@@ -61,6 +61,7 @@ extra_rdoc_files: []
 files:
 - Gemfile
 - Gemfile.lock
+- LICENSE
 - README.md
 - Rakefile
 - fluent-plugin-grok-parser.gemspec
@@ -68,6 +69,7 @@ files:
 - patterns/grok-patterns
 - patterns/nagios
 - test/test_grok_parser.rb
+- test/test_grok_parser_in_tcp.rb
 homepage: https://github.com/kiyoto/fluent-plugin-grok-parser
 licenses:
 - Apache License, Version 2.0
@@ -94,3 +96,4 @@ specification_version: 4
 summary: Fluentd plugin to suppor Logstash-inspired Grok format for parsing logs
 test_files:
 - test/test_grok_parser.rb
+- test/test_grok_parser_in_tcp.rb