fluent-plugin-grok-parser 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5c95b3d25597cea368b25f55f45c222a373097ea
-  data.tar.gz: b42a68ae28f5ffd37bc9be15b511fc0a0d34fcce
+  metadata.gz: 12eb3847dea9ea73c534101a7ab66da42fdde1c9
+  data.tar.gz: a4266d71bffd2d8b090330f998465ca4ca284521
 SHA512:
-  metadata.gz: 64f5933dfbee0819f2a26c63a5fd6c586c9013dbead10f7df0af1d9761f56a66545e808cc06493e987b13d0b5fc4ff8d20bd7c9bc46742c0e4d5c23da5d71434
-  data.tar.gz: 971aeca2d5f7fb2f8c88ffb8370414f67beab55697218d0504ed2997d3a2b117ff73412ae41045e0f82007ad3e38c51cb9889618f516616999c6c52f78e86ac9
+  metadata.gz: 2eaab2b69773e7a6438ea7f2826cf20141b32d1f479ff81019ce0d8a8705bff1398f7585b85a18fba3803b3f42acc60470424899c6d017771b52bb8f9c4b5239
+  data.tar.gz: 38d37a71b73564647b9d1e31000e3c34a75a4bf6c7c45adf7b7412a7924a407cbcde83e56dea9f15badb8b05e0a9527311af698f1b880158d1c254ba2b2ee412

data/README.md

@@ -1,4 +1,4 @@
-# Grok Parser for Fluentd [![Build Status](https://travis-ci.org/kiyoto/fluent-plugin-grok-parser.svg?branch=master)](https://travis-ci.org/kiyoto/fluent-plugin-grok-parser)
+# Grok Parser for Fluentd [![Build Status](https://travis-ci.org/fluent/fluent-plugin-grok-parser.svg?branch=master)](https://travis-ci.org/fluent/fluent-plugin-grok-parser)
 
 This is a Fluentd plugin to enable Logstash's Grok-like parsing logic.
 
@@ -169,7 +169,7 @@ Please see `patterns/*` for the patterns that are supported out of the box.
 You can add your own Grok patterns by creating your own Grok file and telling the plugin to read it.
 This is what the `custom_pattern_path` parameter is for.
 
-```
+```aconf
 <source>
   @type tail
   path /path/to/log
@@ -232,6 +232,8 @@ Here is a sample config using the Grok parser with `in_tail` and the `types` par
 
 If you want to use this plugin with Fluentd v0.12.x or earlier, you can use this plugin version v1.0.0.
 
+See also: [Plugin Management | Fluentd](http://docs.fluentd.org/articles/plugin-management#ldquomdashgemfilerdquo-option)
+
 ## License
 
 Apache 2.0 License

data/Rakefile

@@ -1,11 +1,11 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
-require 'rake/testtask'
-require 'rake/clean'
+require "rake/testtask"
+require "rake/clean"
 
 task :test => [:base_test]
 
-desc 'Run test_unit based test'
+desc "Run test_unit based test"
 Rake::TestTask.new(:base_test) do |t|
   t.libs << "test"
   t.test_files = (Dir["test/test_*.rb"] + Dir["test/plugin/test_*.rb"] - ["helper.rb"]).sort
@@ -13,8 +13,8 @@ Rake::TestTask.new(:base_test) do |t|
   # t.warning = false
 end
 
-desc 'Import patterns from submodules'
-task 'patterns:import' do
+desc "Import patterns from submodules"
+task "patterns:import" do
   `git submodule --quiet foreach pwd`.split($\).each do |submodule_path|
     Dir.glob(File.join(submodule_path, "patterns/*")) do |pattern|
       cp(pattern, "patterns/", verbose: true)

data/fluent-plugin-grok-parser.gemspec

@@ -1,14 +1,14 @@
 # coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-grok-parser"
-  spec.version       = "2.0.0"
+  spec.version       = "2.0.1"
   spec.authors       = ["kiyoto"]
   spec.email         = ["kiyoto@treasure-data.com"]
   spec.summary       = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}
-  spec.homepage      = "https://github.com/kiyoto/fluent-plugin-grok-parser"
+  spec.homepage      = "https://github.com/fluent/fluent-plugin-grok-parser"
   spec.license       = "Apache-2.0"
 
   spec.files         = `git ls-files`.split($/)

data/lib/fluent/plugin/grok.rb

@@ -27,26 +27,26 @@ module Fluent
       if plugin.respond_to?(:firstline?)
         @multiline_mode = true
       end
-      if @conf['multiline_start_regexp']
-        @multiline_start_regexp = Regexp.compile(@conf['multiline_start_regexp'][1..-2])
+      if @conf["multiline_start_regexp"]
+        @multiline_start_regexp = Regexp.compile(@conf["multiline_start_regexp"][1..-2])
       end
     end
 
     def add_patterns_from_file(path)
       File.new(path).each_line do |line|
-        next if line[0] == '#' || /^$/ =~ line
+        next if line[0] == "#" || /^$/ =~ line
         name, pat = line.chomp.split(/\s+/, 2)
         @pattern_map[name] = pat
       end
     end
 
     def setup
-      if @conf['grok_pattern']
-        @parsers << expand_pattern_expression(@conf['grok_pattern'], @conf)
+      if @conf["grok_pattern"]
+        @parsers << expand_pattern_expression(@conf["grok_pattern"], @conf)
       else
-        grok_confs = @conf.elements.select {|e| e.name == 'grok'}
+        grok_confs = @conf.elements.select {|e| e.name == "grok"}
         grok_confs.each do |grok_conf|
-          @parsers << expand_pattern_expression(grok_conf['pattern'], grok_conf)
+          @parsers << expand_pattern_expression(grok_conf["pattern"], grok_conf)
         end
       end
     end
@@ -56,18 +56,18 @@ module Fluent
     def expand_pattern_expression(grok_pattern, conf)
       regexp, types = expand_pattern(grok_pattern)
       $log.info "Expanded the pattern #{conf['grok_pattern']} into #{regexp}"
-      options = nil
-      if @multiline_mode
-        options = Regexp::MULTILINE
-      end
       unless types.empty?
         conf["types"] = types.map{|subname,type| "#{subname}:#{type}" }.join(",")
       end
-      TextParser::RegexpParser.new(Regexp.new(regexp, options), conf)
+      _conf = conf.merge("expression" => regexp, "multiline" => @multiline_mode)
+      config = Fluent::Config::Element.new("parse", nil, _conf, [])
+      parser = Fluent::Plugin::RegexpParser.new
+      parser.configure(config)
+      parser
     rescue GrokPatternNotFoundError => e
       raise e
     rescue => e
-      $log.error e.backtrace.join("\n")
+      $log.error(error: e)
     end
 
     def expand_pattern(pattern)

data/lib/fluent/plugin/parser_grok.rb

@@ -2,17 +2,15 @@ require "fluent/plugin/grok"
 
 module Fluent
   module Plugin
-    class GrokPatternNotFoundError < Exception; end
-
     class GrokParser < Parser
-      Fluent::Plugin.register_parser('grok', self)
+      Fluent::Plugin.register_parser("grok", self)
 
-      desc 'The format of the time field.'
-      config_param :time_format, :string, :default => nil
-      desc 'The pattern of grok'
-      config_param :grok_pattern, :string, :default => nil
-      desc 'Path to the file that includes custom grok patterns'
-      config_param :custom_pattern_path, :string, :default => nil
+      desc "The format of the time field."
+      config_param :time_format, :string, default: nil
+      desc "The pattern of grok"
+      config_param :grok_pattern, :string, default: nil
+      desc "Path to the file that includes custom grok patterns"
+      config_param :custom_pattern_path, :string, default: nil
 
       def initialize
         super
@@ -24,14 +22,14 @@ module Fluent
 
         @grok = Grok.new(self, conf)
 
-        default_pattern_dir = File.expand_path('../../../../patterns/*', __FILE__)
+        default_pattern_dir = File.expand_path("../../../../patterns/*", __FILE__)
         Dir.glob(default_pattern_dir) do |pattern_file_path|
           @grok.add_patterns_from_file(pattern_file_path)
         end
 
         if @custom_pattern_path
           if Dir.exist? @custom_pattern_path
-            Dir.glob(@custom_pattern_path + '/*') do |pattern_file_path|
+            Dir.glob(@custom_pattern_path + "/*") do |pattern_file_path|
               @grok.add_patterns_from_file(pattern_file_path)
             end
           elsif File.exist? @custom_pattern_path
@@ -51,7 +49,9 @@ module Fluent
             end
           end
         end
-        yield @default_parser.parse(text)
+        @default_parser.parse(text) do |time, record|
+          yield time, record
+        end
       end
     end
   end

data/lib/fluent/plugin/parser_multiline_grok.rb

@@ -1,12 +1,12 @@
-require 'fluent/plugin/parser_grok'
+require "fluent/plugin/parser_grok"
 
 module Fluent
   module Plugin
     class MultilineGrokParser < GrokParser
-      Fluent::Plugin.register_parser('multiline_grok', self)
+      Fluent::Plugin.register_parser("multiline_grok", self)
 
-      desc 'The regexp to match beginning of multiline'
-      config_param :multiline_start_regexp, :string, :default => nil
+      desc "The regexp to match beginning of multiline"
+      config_param :multiline_start_regexp, :string, default: nil
 
       def initialize
         super

data/test/helper.rb

@@ -1,6 +1,10 @@
+require "fluent/test"
+require "fluent/test/helpers"
 require "fluent/test/driver/input"
 require "fluent/test/driver/parser"
 
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+
 def unused_port
   s = TCPServer.open(0)
   port = s.addr[1]
@@ -9,7 +13,7 @@ def unused_port
 end
 
 def ipv6_enabled?
-  require 'socket'
+  require "socket"
 
   begin
     TCPServer.open("::1", 0)

data/test/test_grok_parser.rb

@@ -1,10 +1,6 @@
-require 'fluent/test'
-require 'fluent/parser'
-require 'fluent/plugin/parser_grok'
-require 'tempfile'
-
-
-include Fluent
+require "helper"
+require "tempfile"
+require "fluent/plugin/parser_grok"
 
 def str2time(str_time, format = nil)
   if format
@@ -17,40 +13,40 @@ end
 class GrokParserTest < ::Test::Unit::TestCase
   class Timestamp < self
     def test_timestamp_iso8601
-      internal_test_grok_pattern('%{TIMESTAMP_ISO8601:time}', 'Some stuff at 2014-01-01T00:00:00+0900',
-                                 str2time('2014-01-01T00:00:00+0900'), {})
+      internal_test_grok_pattern("%{TIMESTAMP_ISO8601:time}", "Some stuff at 2014-01-01T00:00:00+0900",
+                                 event_time("2014-01-01T00:00:00+0900"), {})
     end
 
     def test_datestamp_rfc822_with_zone
-      internal_test_grok_pattern('%{DATESTAMP_RFC822:time}', 'Some stuff at Mon Aug 15 2005 15:52:01 UTC',
-                                 str2time('Mon Aug 15 2005 15:52:01 UTC'), {})
+      internal_test_grok_pattern("%{DATESTAMP_RFC822:time}", "Some stuff at Mon Aug 15 2005 15:52:01 UTC",
+                                 event_time("Mon Aug 15 2005 15:52:01 UTC"), {})
     end
 
     def test_datestamp_rfc822_with_numeric_zone
-      internal_test_grok_pattern('%{DATESTAMP_RFC2822:time}', 'Some stuff at Mon, 15 Aug 2005 15:52:01 +0000',
-                                 str2time('Mon, 15 Aug 2005 15:52:01 +0000'), {})
+      internal_test_grok_pattern("%{DATESTAMP_RFC2822:time}", "Some stuff at Mon, 15 Aug 2005 15:52:01 +0000",
+                                 event_time("Mon, 15 Aug 2005 15:52:01 +0000"), {})
     end
 
     def test_syslogtimestamp
-      internal_test_grok_pattern('%{SYSLOGTIMESTAMP:time}', 'Some stuff at Aug 01 00:00:00',
-                                 str2time('Aug 01 00:00:00'), {})
+      internal_test_grok_pattern("%{SYSLOGTIMESTAMP:time}", "Some stuff at Aug 01 00:00:00",
+                                 event_time("Aug 01 00:00:00"), {})
     end
   end
 
   def test_call_for_grok_pattern_not_found
-    assert_raise Grok::GrokPatternNotFoundError do
-      internal_test_grok_pattern('%{THIS_PATTERN_DOESNT_EXIST}', 'Some stuff at somewhere', nil, {})
+    assert_raise Fluent::Grok::GrokPatternNotFoundError do
+      internal_test_grok_pattern("%{THIS_PATTERN_DOESNT_EXIST}", "Some stuff at somewhere", nil, {})
     end
   end
 
   def test_call_for_multiple_fields
-    internal_test_grok_pattern('%{MAC:mac_address} %{IP:ip_address}', 'this.wont.match DEAD.BEEF.1234 127.0.0.1', nil,
+    internal_test_grok_pattern("%{MAC:mac_address} %{IP:ip_address}", "this.wont.match DEAD.BEEF.1234 127.0.0.1", nil,
                                {"mac_address" => "DEAD.BEEF.1234", "ip_address" => "127.0.0.1"})
   end
 
   def test_call_for_complex_pattern
-    internal_test_grok_pattern('%{COMBINEDAPACHELOG}', '127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"',
-                                str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'),
+    internal_test_grok_pattern("%{COMBINEDAPACHELOG}", '127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"',
+                                str2time("28/Feb/2013:12:00:00 +0900", "%d/%b/%Y:%H:%M:%S %z"),
                                 {
                                   "clientip"    => "127.0.0.1",
                                   "ident"       => "192.168.0.1",
@@ -73,7 +69,7 @@ class GrokParserTest < ::Test::Unit::TestCase
     pattern_file.write("MY_AWESOME_PATTERN %{GREEDYDATA:message}\n")
     pattern_file.close
     begin
-      internal_test_grok_pattern('%{MY_AWESOME_PATTERN:message}', 'this is awesome',
+      internal_test_grok_pattern("%{MY_AWESOME_PATTERN:message}", "this is awesome",
                                  nil, {"message" => "this is awesome"},
                                  "custom_pattern_path" => pattern_file.path
                                 )
@@ -84,53 +80,53 @@ class GrokParserTest < ::Test::Unit::TestCase
 
   class OptionalType < self
     def test_simple
-      internal_test_grok_pattern('%{INT:user_id:integer} paid %{NUMBER:paid_amount:float}',
-                                 '12345 paid 6789.10', nil,
+      internal_test_grok_pattern("%{INT:user_id:integer} paid %{NUMBER:paid_amount:float}",
+                                 "12345 paid 6789.10", nil,
                                  {"user_id" => 12345, "paid_amount" => 6789.1 })
     end
 
     def test_array
-      internal_test_grok_pattern('%{GREEDYDATA:message:array}',
-                                 'a,b,c,d', nil,
+      internal_test_grok_pattern("%{GREEDYDATA:message:array}",
+                                 "a,b,c,d", nil,
                                  {"message" => %w(a b c d)})
     end
 
     def test_array_with_delimiter
-      internal_test_grok_pattern('%{GREEDYDATA:message:array:|}',
-                                 'a|b|c|d', nil,
+      internal_test_grok_pattern("%{GREEDYDATA:message:array:|}",
+                                 "a|b|c|d", nil,
                                  {"message" => %w(a b c d)})
     end
 
     def test_timestamp_iso8601
-      internal_test_grok_pattern('%{TIMESTAMP_ISO8601:stamp:time}', 'Some stuff at 2014-01-01T00:00:00+0900',
-                                 nil, {"stamp" => str2time('2014-01-01T00:00:00+0900')})
+      internal_test_grok_pattern("%{TIMESTAMP_ISO8601:stamp:time}", "Some stuff at 2014-01-01T00:00:00+0900",
+                                 nil, {"stamp" => event_time("2014-01-01T00:00:00+0900")})
     end
 
     def test_datestamp_rfc822_with_zone
-      internal_test_grok_pattern('%{DATESTAMP_RFC822:stamp:time}', 'Some stuff at Mon Aug 15 2005 15:52:01 UTC',
-                                 nil, {"stamp" => str2time('Mon Aug 15 2005 15:52:01 UTC')})
+      internal_test_grok_pattern("%{DATESTAMP_RFC822:stamp:time}", "Some stuff at Mon Aug 15 2005 15:52:01 UTC",
+                                 nil, {"stamp" => event_time("Mon Aug 15 2005 15:52:01 UTC")})
     end
 
     def test_datestamp_rfc822_with_numeric_zone
-      internal_test_grok_pattern('%{DATESTAMP_RFC2822:stamp:time}', 'Some stuff at Mon, 15 Aug 2005 15:52:01 +0000',
-                                 nil, {"stamp" => str2time('Mon, 15 Aug 2005 15:52:01 +0000')})
+      internal_test_grok_pattern("%{DATESTAMP_RFC2822:stamp:time}", "Some stuff at Mon, 15 Aug 2005 15:52:01 +0000",
+                                 nil, {"stamp" => event_time("Mon, 15 Aug 2005 15:52:01 +0000")})
     end
 
     def test_syslogtimestamp
-      internal_test_grok_pattern('%{SYSLOGTIMESTAMP:stamp:time}', 'Some stuff at Aug 01 00:00:00',
-                                 nil, {"stamp" => str2time('Aug 01 00:00:00')})
+      internal_test_grok_pattern("%{SYSLOGTIMESTAMP:stamp:time}", "Some stuff at Aug 01 00:00:00",
+                                 nil, {"stamp" => event_time("Aug 01 00:00:00")})
     end
 
     def test_timestamp_with_format
-      internal_test_grok_pattern('%{TIMESTAMP_ISO8601:stamp:time:%Y-%m-%d %H%M}', 'Some stuff at 2014-01-01 1000',
-                                 nil, {"stamp" => str2time('2014-01-01 10:00')})
+      internal_test_grok_pattern("%{TIMESTAMP_ISO8601:stamp:time:%Y-%m-%d %H%M}", "Some stuff at 2014-01-01 1000",
+                                 nil, {"stamp" => event_time("2014-01-01 10:00")})
     end
   end
 
   private
 
   def internal_test_grok_pattern(grok_pattern, text, expected_time, expected_record, options = {})
-    d  = Fluent::Test::Driver::Parser.new(Fluent::Plugin::GrokParser).configure({"grok_pattern" => grok_pattern}.merge(options))
+    d = Fluent::Test::Driver::Parser.new(Fluent::Plugin::GrokParser).configure({"grok_pattern" => grok_pattern}.merge(options))
 
     # for the new API
     d.instance.parse(text) {|time, record|

data/test/test_grok_parser_in_tcp.rb

@@ -1,6 +1,5 @@
-require 'helper'
-require 'fluent/test'
-require 'fluent/plugin/in_tcp'
+require "helper"
+require "fluent/plugin/in_tcp"
 
 class TcpInputWithGrokTest < Test::Unit::TestCase
   if defined?(ServerEngine)
@@ -8,7 +7,7 @@ class TcpInputWithGrokTest < Test::Unit::TestCase
       def startup
         socket_manager_path = ServerEngine::SocketManager::Server.generate_path
         @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
-        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+        ENV["SERVERENGINE_SOCKETMANAGER_PATH"] = socket_manager_path.to_s
       end
 
       def shutdown
@@ -39,8 +38,8 @@ class TcpInputWithGrokTest < Test::Unit::TestCase
   end
 
   def test_configure
-    configs = {'127.0.0.1' => CONFIG}
-    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+    configs = {"127.0.0.1" => CONFIG}
+    configs.merge!("::1" => IPv6_CONFIG) if ipv6_enabled?
 
     configs.each_pair { |k, v|
       d = create_driver(v)
@@ -52,8 +51,8 @@ class TcpInputWithGrokTest < Test::Unit::TestCase
 
   def test_grok_pattern
     tests = [
-      {'msg' => "tcptest1\n", 'expected' => 'tcptest1'},
-      {'msg' => "tcptest2\n", 'expected' => 'tcptest2'},
+      {"msg" => "tcptest1\n", "expected" => "tcptest1"},
+      {"msg" => "tcptest2\n", "expected" => "tcptest2"},
     ]
     block_config = %[
       <grok>
@@ -61,14 +60,14 @@ class TcpInputWithGrokTest < Test::Unit::TestCase
       </grok>
     ]
 
-    internal_test_grok('grok_pattern %{GREEDYDATA:message}', tests)
+    internal_test_grok("grok_pattern %{GREEDYDATA:message}", tests)
     internal_test_grok(block_config, tests)
   end
 
   def test_grok_multi_patterns
     tests = [
-      {'msg' => "Current time is 2014-01-01T00:00:00+0900\n", 'expected' => '2014-01-01T00:00:00+0900'},
-      {'msg' => "The first word matches\n", 'expected' => 'The'}
+      {"msg" => "Current time is 2014-01-01T00:00:00+0900\n", "expected" => "2014-01-01T00:00:00+0900"},
+      {"msg" => "The first word matches\n", "expected" => "The"}
     ]
     block_config = %[
       <grok>
@@ -85,8 +84,8 @@ class TcpInputWithGrokTest < Test::Unit::TestCase
     d = create_driver(BASE_CONFIG + conf)
     d.run(expect_emits: tests.size) do
       tests.each {|test|
-        TCPSocket.open('127.0.0.1', PORT) do |s|
-          s.send(test['msg'], 0)
+        TCPSocket.open("127.0.0.1", PORT) do |s|
+          s.send(test["msg"], 0)
         end
       }
     end
@@ -97,7 +96,7 @@ class TcpInputWithGrokTest < Test::Unit::TestCase
   def compare_test_result(events, tests)
     assert_equal(2, events.size)
     events.each_index {|i|
-      assert_equal(tests[i]['expected'], events[i][2]['message'])
+      assert_equal(tests[i]["expected"], events[i][2]["message"])
     }
   end
 end

data/test/test_multiline_grok_parser.rb

@@ -1,7 +1,8 @@
-require 'fluent/plugin/parser_multiline_grok'
-require 'fluent/config/parser'
+require "helper"
+require "fluent/plugin/parser_multiline_grok"
+require "fluent/config/parser"
 
-require 'stringio'
+require "stringio"
 
 class MultilineGrokParserTest < Test::Unit::TestCase
   def test_multiline

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grok-parser
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - kiyoto
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-05 00:00:00.000000000 Z
+date: 2016-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -109,7 +109,7 @@ files:
 - test/test_grok_parser.rb
 - test/test_grok_parser_in_tcp.rb
 - test/test_multiline_grok_parser.rb
-homepage: https://github.com/kiyoto/fluent-plugin-grok-parser
+homepage: https://github.com/fluent/fluent-plugin-grok-parser
 licenses:
 - Apache-2.0
 metadata: {}