fluent-plugin-grok-parser 2.0.1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12eb3847dea9ea73c534101a7ab66da42fdde1c9
-  data.tar.gz: a4266d71bffd2d8b090330f998465ca4ca284521
+  metadata.gz: 2c5c25e59b231fe7a4902d8e691fb1b999185faa
+  data.tar.gz: 2aa97bc6af77717e3833292b040d0669b78f07a7
 SHA512:
-  metadata.gz: 2eaab2b69773e7a6438ea7f2826cf20141b32d1f479ff81019ce0d8a8705bff1398f7585b85a18fba3803b3f42acc60470424899c6d017771b52bb8f9c4b5239
-  data.tar.gz: 38d37a71b73564647b9d1e31000e3c34a75a4bf6c7c45adf7b7412a7924a407cbcde83e56dea9f15badb8b05e0a9527311af698f1b880158d1c254ba2b2ee412
+  metadata.gz: 3f28d4fe0a807b0efde2781e5e0c086689dfa4c56b47722100b7e44088801f9ae342c2dcc7e658405fef0683f38ff8445f9ee65125ccbc3ffebfceb3fdaed74e
+  data.tar.gz: 30ab878638d27d96bfbcdba1072f7458e99210df286aca12f2aa8e9f2ed79bc17bbad04d054bb11bbca9389fb7360307aa43bf1bd304bfa7236058dd934ca58f

data/.travis.yml

@@ -4,5 +4,5 @@ language: ruby
 rvm:
   - 2.1
   - 2.2
-  - 2.3.0
+  - 2.3.1
 

data/README.md

@@ -145,6 +145,78 @@ Fluentd accumulates data in the buffer forever to parse complete data when no pa
 
 You can use this parser without `multiline_start_regexp` when you know your data structure perfectly.
 
+## Configurations
+
+**time_format**
+
+The format of the time field.
+
+**grok_pattern**
+
+The pattern of grok. You cannot specify multiple grok pattern with this.
+
+**custom_pattern_path**
+
+Path to the file that includes custom grok patterns
+
+**grok_failure_key**
+
+The key has grok failure reason. Default is `nil`.
+
+```aconf
+<source>
+  @type dummy
+  @label @dummy
+  dummy [
+    { "message1": "no grok pattern matched!", "prog": "foo" },
+    { "message1": "/", "prog": "bar" }
+  ]
+  tag dummy.log
+</source>
+
+<label @dummy>
+  <filter>
+    @type parser
+    key_name message1
+    reserve_data true
+    reserve_time true
+    <parse>
+      @type grok
+      grok_failure_key grokfailure
+      <grok>
+        pattern %{PATH:path}
+      </grok>
+    </parse>
+  </filter>
+  <match dummy.log>
+    @type stdout
+  </match>
+</label>
+```
+
+This generates following events:
+
+```
+2016-11-28 13:07:08.009131727 +0900 dummy.log: {"message1":"no grok pattern matched!","prog":"foo","message":"no grok pattern matched!","grokfailure":"No grok pattern matched"}
+2016-11-28 13:07:09.010400923 +0900 dummy.log: {"message1":"/","prog":"bar","path":"/"}
+```
+
+
+**grok/pattern**
+
+Section for grok patterns. You can use multiple grok patterns with
+multiple `<grok>` sections.
+
+```aconf
+<grok>
+  pattern %{IP:ipaddress}
+</grok>
+```
+
+**multiline_start_regexp**
+
+The regexp to match beginning of multiline. This is only for "multiline_grok".
+
 ## How to write Grok patterns
 
 Grok patterns look like `%{PATTERN_NAME:name}` where ":name" is optional. If "name" is provided, then it

data/fluent-plugin-grok-parser.gemspec

@@ -4,9 +4,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-grok-parser"
-  spec.version       = "2.0.1"
-  spec.authors       = ["kiyoto"]
-  spec.email         = ["kiyoto@treasure-data.com"]
+  spec.version       = "2.1.0"
+  spec.authors       = ["kiyoto", "Kenji Okimoto"]
+  spec.email         = ["kiyoto@treasure-data.com", "okimoto@clear-code.com"]
   spec.summary       = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}
   spec.homepage      = "https://github.com/fluent/fluent-plugin-grok-parser"
   spec.license       = "Apache-2.0"

data/lib/fluent/plugin/grok.rb

@@ -33,7 +33,7 @@ module Fluent
     end
 
     def add_patterns_from_file(path)
-      File.new(path).each_line do |line|
+      File.open(path, "r").each_line do |line|
         next if line[0] == "#" || /^$/ =~ line
         name, pat = line.chomp.split(/\s+/, 2)
         @pattern_map[name] = pat

data/lib/fluent/plugin/parser_grok.rb

@@ -11,6 +11,8 @@ module Fluent
       config_param :grok_pattern, :string, default: nil
       desc "Path to the file that includes custom grok patterns"
       config_param :custom_pattern_path, :string, default: nil
+      desc "The key has grok failure reason"
+      config_param :grok_failure_key, :string, default: nil
 
       def initialize
         super
@@ -50,6 +52,7 @@ module Fluent
           end
         end
         @default_parser.parse(text) do |time, record|
+          record[@grok_failure_key] = "No grok pattern matched" if @grok_failure_key
           yield time, record
         end
       end

data/lib/fluent/plugin/parser_multiline_grok.rb

@@ -8,14 +8,6 @@ module Fluent
       desc "The regexp to match beginning of multiline"
       config_param :multiline_start_regexp, :string, default: nil
 
-      def initialize
-        super
-      end
-
-      def configure(conf={})
-        super
-      end
-
       def has_firstline?
         !!@multiline_start_regexp
       end
@@ -33,6 +25,10 @@ module Fluent
             end
           end
         end
+        @default_parser.parse(text) do |time, record|
+          record[@grok_failure_key] = "No grok pattern matched" if @grok_failure_key
+          yield time, record
+        end
       end
     end
   end

data/test/test_grok_parser.rb

@@ -123,10 +123,48 @@ class GrokParserTest < ::Test::Unit::TestCase
     end
   end
 
+  class NoGrokPatternMatched < self
+    def test_with_grok_failure_key
+      config = %[
+        grok_failure_key grok_failure
+        <grok>
+          pattern %{PATH:path}
+        </grok>
+      ]
+      expected = {
+        "grok_failure" => "No grok pattern matched",
+        "message" => "no such pattern"
+      }
+      d = create_driver(config)
+      d.instance.parse("no such pattern") do |_time, record|
+        assert_equal(expected, record)
+      end
+    end
+
+    def test_without_grok_failure_key
+      config = %[
+        <grok>
+          pattern %{PATH:path}
+        </grok>
+      ]
+      expected = {
+        "message" => "no such pattern"
+      }
+      d = create_driver(config)
+      d.instance.parse("no such pattern") do |_time, record|
+        assert_equal(expected, record)
+      end
+    end
+  end
+
   private
 
+  def create_driver(conf)
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::GrokParser).configure(conf)
+  end
+
   def internal_test_grok_pattern(grok_pattern, text, expected_time, expected_record, options = {})
-    d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::GrokParser).configure({"grok_pattern" => grok_pattern}.merge(options))
+    d = create_driver({"grok_pattern" => grok_pattern}.merge(options))
 
     # for the new API
     d.instance.parse(text) {|time, record|

data/test/test_multiline_grok_parser.rb

@@ -61,6 +61,40 @@ TEXT
     assert(d.instance.firstline?(text))
   end
 
+  class NoGrokPatternMatched < self
+    def test_with_grok_failure_key
+      config = %[
+        grok_failure_key grok_failure
+        <grok>
+          pattern %{PATH:path}
+        </grok>
+      ]
+      expected = {
+        "grok_failure" => "No grok pattern matched",
+        "message" => "no such pattern\nno such pattern\n"
+      }
+      d = create_driver(config)
+      d.instance.parse("no such pattern\nno such pattern\n") do |_time, record|
+        assert_equal(expected, record)
+      end
+    end
+
+    def test_without_grok_failure_key
+      config = %[
+        <grok>
+          pattern %{PATH:path}
+        </grok>
+      ]
+      expected = {
+        "message" => "no such pattern\nno such pattern\n"
+      }
+      d = create_driver(config)
+      d.instance.parse("no such pattern\nno such pattern\n") do |_time, record|
+        assert_equal(expected, record)
+      end
+    end
+  end
+
   private
 
   def create_driver(conf)

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grok-parser
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.1.0
 platform: ruby
 authors:
 - kiyoto
+- Kenji Okimoto
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-25 00:00:00.000000000 Z
+date: 2016-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -69,6 +70,7 @@ dependencies:
 description: 
 email:
 - kiyoto@treasure-data.com
+- okimoto@clear-code.com
 executables: []
 extensions: []
 extra_rdoc_files: []