fluent-plugin-grok-parser 1.0.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: abba93dea7171b49379a1c09b97e93775a71c60f07975d8be225467ef9a02f3e
-  data.tar.gz: 53cabdd496599b84fda4a5facebbd93b59be9a5426dfa3142f6b94344411431d
+SHA1:
+  metadata.gz: 5c95b3d25597cea368b25f55f45c222a373097ea
+  data.tar.gz: b42a68ae28f5ffd37bc9be15b511fc0a0d34fcce
 SHA512:
-  metadata.gz: 0fed7231df5bddfe56b57eaa53155afb29e400d16994e3454d80a1f7dde75ff0528fd778137a65d557d71529b1de75f501206f3dfd5220cb5b2b2e86fd44d8ad
-  data.tar.gz: 37e00405e9c28e9d4bac1fe02ea609e297ea98554d98802f8deff575d5c8104953149dbbc129b00953fb5e162390904af2cf707df76f073bae3e608594fe2052
+  metadata.gz: 64f5933dfbee0819f2a26c63a5fd6c586c9013dbead10f7df0af1d9761f56a66545e808cc06493e987b13d0b5fc4ff8d20bd7c9bc46742c0e4d5c23da5d71434
+  data.tar.gz: 971aeca2d5f7fb2f8c88ffb8370414f67beab55697218d0504ed2997d3a2b117ff73412ae41045e0f82007ad3e38c51cb9889618f516616999c6c52f78e86ac9

data/README.md

@@ -17,9 +17,23 @@ extracts the first IP address that matches in the log.
 <source>
   @type tail
   path /path/to/log
+  tag grokked_log
+  <parse>
+    @type grok
+    grok_pattern %{IP:ip_address}
+  </parse>
+</source>
+```
+
+You can also use Fluentd v0.12 style:
+
+```aconf
+<source>
+  @type tail
+  path /path/to/log
+  tag grokked_log
   format grok
   grok_pattern %{IP:ip_address}
-  tag grokked_log
 </source>
 ```
 
@@ -29,6 +43,30 @@ extracts the first IP address that matches in the log.
 <source>
   @type tail
   path /path/to/log
+  tag grokked_log
+  <parse>
+    @type grok
+    <grok>
+      pattern %{COMBINEDAPACHELOG}
+      time_format "%d/%b/%Y:%H:%M:%S %z"
+    </grok>
+    <grok>
+      pattern %{IP:ip_address}
+    </grok>
+    <grok>
+      pattern %{GREEDYDATA:message}
+    </grok>
+  </parse>
+</source>
+```
+
+You can also use Fluentd v0.12 style:
+
+```aconf
+<source>
+  @type tail
+  path /path/to/log
+  tag grokked_log
   format grok
   <grok>
     pattern %{COMBINEDAPACHELOG}
@@ -40,7 +78,6 @@ extracts the first IP address that matches in the log.
   <grok>
     pattern %{GREEDYDATA:message}
   </grok>
-  tag grokked_log
 </source>
 ```
 
@@ -48,6 +85,21 @@ extracts the first IP address that matches in the log.
 
 You can parse multiple line text.
 
+```aconf
+<source>
+  @type tail
+  path /path/to/log
+  tag grokked_log
+  <parse>
+    @type multiline_grok
+    grok_pattern %{IP:ip_address}%{GREEDYDATA:message}
+    multiline_start_regexp /^[^\s]/
+  </parse>
+</source>
+```
+
+You can also use Fluentd v0.12 style:
+
 ```aconf
 <source>
   @type tail
@@ -61,6 +113,22 @@ You can parse multiple line text.
 
 You can use multiple grok patterns to parse your data.
 
+```aconf
+<source>
+  @type tail
+  path /path/to/log
+  tag grokked_log
+  <parse>
+    @type multiline_grok
+    <grok>
+      pattern Started %{WORD:verb} "%{URIPATH:pathinfo}" for %{IP:ip} at %{TIMESTAMP_ISO8601:timestamp}\nProcessing by %{WORD:controller}#%{WORD:action} as %{WORD:format}%{DATA:message}Completed %{NUMBER:response} %{WORD} in %{NUMBER:elapsed} (%{DATA:elapsed_details})
+    </grok>
+  </parse>
+</source>
+```
+
+You can also use Fluentd v0.12 style:
+
 ```aconf
 <source>
   @type tail
@@ -105,9 +173,11 @@ This is what the `custom_pattern_path` parameter is for.
 <source>
   @type tail
   path /path/to/log
-  format grok
-  grok_pattern %{MY_SUPER_PATTERN}
-  custom_pattern_path /path/to/my_pattern
+  <parse>
+    @type grok
+    grok_pattern %{MY_SUPER_PATTERN}
+    custom_pattern_path /path/to/my_pattern
+  </parse>
 </source>
 ```
 
@@ -158,6 +228,10 @@ Here is a sample config using the Grok parser with `in_tail` and the `types` par
 </source>
 ```
 
+## Notice
+
+If you want to use this plugin with Fluentd v0.12.x or earlier, you can use this plugin version v1.0.0.
+
 ## License
 
 Apache 2.0 License

data/Rakefile

@@ -9,8 +9,8 @@ desc 'Run test_unit based test'
 Rake::TestTask.new(:base_test) do |t|
   t.libs << "test"
   t.test_files = (Dir["test/test_*.rb"] + Dir["test/plugin/test_*.rb"] - ["helper.rb"]).sort
-  t.verbose = false
-  t.warning = false
+  t.verbose = true
+  # t.warning = false
 end
 
 desc 'Import patterns from submodules'
@@ -20,42 +20,6 @@ task 'patterns:import' do
       cp(pattern, "patterns/", verbose: true)
     end
   end
-
-  # copied from "./lib/fluent/plugin/grok"
-  pattern_re =
-    /%\{    # match '%{' not prefixed with '\'
-      (?<name>     # match the pattern name
-        (?<pattern>[A-z0-9]+)
-        (?::(?<subname>[@\[\]A-z0-9_:.-]+?)
-             (?::(?<type>(?:string|bool|integer|float|int|
-                            time(?::.+)?|
-                            array(?::.)?)))?)?
-      )
-    \}/x
-
-  Dir.glob("patterns/*") do |pattern_file|
-    new_lines = ""
-    File.readlines(pattern_file).each do |line|
-      case
-      when line.strip.empty?
-        new_lines << line
-      when line.start_with?("#")
-        new_lines << line
-      else
-        name, pattern = line.split(/\s+/, 2)
-        new_pattern = pattern.gsub(pattern_re) do |m|
-          matched = $~
-          if matched[:type] == "int"
-            "%{#{matched[:pattern]}:#{matched[:subname]}:integer}"
-          else
-            m
-          end
-        end
-        new_lines << "#{name} #{new_pattern}"
-      end
-    end
-    File.write(pattern_file, new_lines)
-  end
 end
 
 task :default => [:test, :build]

data/fluent-plugin-grok-parser.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-grok-parser"
-  spec.version       = "1.0.1"
+  spec.version       = "2.0.0"
   spec.authors       = ["kiyoto"]
   spec.email         = ["kiyoto@treasure-data.com"]
   spec.summary       = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}
@@ -19,5 +19,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit", ">=3.1.5"
-  spec.add_runtime_dependency "fluentd", [">=0.10.58", "~>0.12.0"]
+  spec.add_runtime_dependency "fluentd", ">=0.14.6"
 end

data/lib/fluent/plugin/grok.rb

@@ -82,7 +82,7 @@ module Fluent
           replacement_pattern = "(?<#{m["subname"]}>#{curr_pattern})"
           type_map[m["subname"]] = m["type"] || "string"
         else
-          replacement_pattern = "(?:#{curr_pattern})"
+          replacement_pattern = curr_pattern
         end
         pattern.sub!(m[0]) do |s| replacement_pattern end
       end

data/lib/fluent/plugin/parser_grok.rb

@@ -1,19 +1,11 @@
 require "fluent/plugin/grok"
 
 module Fluent
-  class TextParser
+  module Plugin
     class GrokPatternNotFoundError < Exception; end
 
     class GrokParser < Parser
-      Plugin.register_parser('grok', self)
-
-      # For fluentd v0.12.16 or earlier
-      class << self
-        unless method_defined?(:desc)
-          def desc(description)
-          end
-        end
-      end
+      Fluent::Plugin.register_parser('grok', self)
 
       desc 'The format of the time field.'
       config_param :time_format, :string, :default => nil
@@ -38,11 +30,11 @@ module Fluent
         end
 
         if @custom_pattern_path
-          if Dir.exists? @custom_pattern_path
+          if Dir.exist? @custom_pattern_path
             Dir.glob(@custom_pattern_path + '/*') do |pattern_file_path|
               @grok.add_patterns_from_file(pattern_file_path)
             end
-          elsif File.exists? @custom_pattern_path
+          elsif File.exist? @custom_pattern_path
             @grok.add_patterns_from_file(@custom_pattern_path)
           end
         end
@@ -50,27 +42,16 @@ module Fluent
         @grok.setup
       end
 
-      def parse(text, &block)
-        if block_given?
-          @grok.parsers.each do |parser|
-            parser.parse(text) do |time, record|
-              if time and record
-                yield time, record
-                return
-              end
-            end
-          end
-          yield @default_parser.parse(text)
-        else
-          @grok.parsers.each do |parser|
-            parser.parse(text) do |time, record|
-              if time and record
-                return time, record
-              end
+      def parse(text)
+        @grok.parsers.each do |parser|
+          parser.parse(text) do |time, record|
+            if time and record
+              yield time, record
+              return
             end
           end
-          return @default_parser.parse(text)
         end
+        yield @default_parser.parse(text)
       end
     end
   end

data/lib/fluent/plugin/parser_multiline_grok.rb

@@ -1,9 +1,9 @@
 require 'fluent/plugin/parser_grok'
 
 module Fluent
-  class TextParser
+  module Plugin
     class MultilineGrokParser < GrokParser
-      Plugin.register_parser('multiline_grok', self)
+      Fluent::Plugin.register_parser('multiline_grok', self)
 
       desc 'The regexp to match beginning of multiline'
       config_param :multiline_start_regexp, :string, :default => nil
@@ -24,22 +24,12 @@ module Fluent
         @multiline_start_regexp && !!@grok.multiline_start_regexp.match(text)
       end
 
-      def parse(text, &block)
-        if block_given?
-          @grok.parsers.each do |parser|
-            parser.parse(text) do |time, record|
-              if time and record
-                yield time, record
-                return
-              end
-            end
-          end
-        else
-          @grok.parsers.each do |parser|
-            parser.parse(text) do |time, record|
-              if time and record
-                return time, record
-              end
+      def parse(text)
+        @grok.parsers.each do |parser|
+          parser.parse(text) do |time, record|
+            if time and record
+              yield time, record
+              return
             end
           end
         end

data/patterns/aws

@@ -1,6 +1,6 @@
 S3_REQUEST_LINE (?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})
 
-S3_ACCESS_LOG %{WORD:owner} %{NOTSPACE:bucket} \[%{HTTPDATE:timestamp}\] %{IP:clientip} %{NOTSPACE:requester} %{NOTSPACE:request_id} %{NOTSPACE:operation} %{NOTSPACE:key} (?:"%{S3_REQUEST_LINE}"|-) (?:%{INT:response:integer}|-) (?:-|%{NOTSPACE:error_code}) (?:%{INT:bytes:integer}|-) (?:%{INT:object_size:integer}|-) (?:%{INT:request_time_ms:integer}|-) (?:%{INT:turnaround_time_ms:integer}|-) (?:%{QS:referrer}|-) (?:"?%{QS:agent}"?|-) (?:-|%{NOTSPACE:version_id})
+S3_ACCESS_LOG %{WORD:owner} %{NOTSPACE:bucket} \[%{HTTPDATE:timestamp}\] %{IP:clientip} %{NOTSPACE:requester} %{NOTSPACE:request_id} %{NOTSPACE:operation} %{NOTSPACE:key} (?:"%{S3_REQUEST_LINE}"|-) (?:%{INT:response:int}|-) (?:-|%{NOTSPACE:error_code}) (?:%{INT:bytes:int}|-) (?:%{INT:object_size:int}|-) (?:%{INT:request_time_ms:int}|-) (?:%{INT:turnaround_time_ms:int}|-) (?:%{QS:referrer}|-) (?:"?%{QS:agent}"?|-) (?:-|%{NOTSPACE:version_id})
 
 ELB_URIPATHPARAM %{URIPATH:path}(?:%{URIPARAM:params})?
 
@@ -8,7 +8,7 @@ ELB_URI %{URIPROTO:proto}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{
 
 ELB_REQUEST_LINE (?:%{WORD:verb} %{ELB_URI:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})
 
-ELB_ACCESS_LOG %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:integer} (?:(%{IP:backendip}:?:%{INT:backendport:integer})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:integer} %{INT:backend_response:integer} %{INT:received_bytes:integer} %{INT:bytes:integer} "%{ELB_REQUEST_LINE}"
+ELB_ACCESS_LOG %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:int} (?:(%{IP:backendip}:?:%{INT:backendport:int})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:int} %{INT:backend_response:int} %{INT:received_bytes:int} %{INT:bytes:int} "%{ELB_REQUEST_LINE}"
 
-CLOUDFRONT_ACCESS_LOG (?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\t%{TIME})\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:integer}|-)\t%{IPORHOST:clientip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:cs_uri_stem}\t%{NUMBER:sc_status:integer}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:agent}\t%{GREEDYDATA:cs_uri_query}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:integer}\t%{GREEDYDATA:time_taken:float}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}
+CLOUDFRONT_ACCESS_LOG (?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\t%{TIME})\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:int}|-)\t%{IPORHOST:clientip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:cs_uri_stem}\t%{NUMBER:sc_status:int}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:agent}\t%{GREEDYDATA:cs_uri_query}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:int}\t%{GREEDYDATA:time_taken:float}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}
 

data/test/helper.rb

@@ -1,3 +1,5 @@
+require "fluent/test/driver/input"
+require "fluent/test/driver/parser"
 
 def unused_port
   s = TCPServer.open(0)

data/test/test_grok_parser.rb

@@ -37,11 +37,6 @@ class GrokParserTest < ::Test::Unit::TestCase
     end
   end
 
-  def test_date
-    internal_test_grok_pattern("\\[(?<date>%{DATE} %{TIME} (?:AM|PM))\\]", "[2/16/2018 10:19:34 AM]",
-                               nil, { "date" => "2/16/2018 10:19:34 AM" })
-  end
-
   def test_call_for_grok_pattern_not_found
     assert_raise Grok::GrokPatternNotFoundError do
       internal_test_grok_pattern('%{THIS_PATTERN_DOESNT_EXIST}', 'Some stuff at somewhere', nil, {})
@@ -135,15 +130,10 @@ class GrokParserTest < ::Test::Unit::TestCase
   private
 
   def internal_test_grok_pattern(grok_pattern, text, expected_time, expected_record, options = {})
-    parser = Fluent::Test::ParserTestDriver.new(TextParser::GrokParser).configure({"grok_pattern" => grok_pattern}.merge(options))
-
-    # for the old, return based API
-    time, record = parser.parse(text)
-    assert_equal(expected_time, time) if expected_time
-    assert_equal(expected_record, record)
+    d  = Fluent::Test::Driver::Parser.new(Fluent::Plugin::GrokParser).configure({"grok_pattern" => grok_pattern}.merge(options))
 
     # for the new API
-    parser.parse(text) {|time, record|
+    d.instance.parse(text) {|time, record|
       assert_equal(expected_time, time) if expected_time
       assert_equal(expected_record, record)
     }

data/test/test_grok_parser_in_tcp.rb

@@ -35,7 +35,7 @@ class TcpInputWithGrokTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf)
-    Fluent::Test::InputTestDriver.new(Fluent::TcpInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::TcpInput).configure(conf)
   end
 
   def test_configure
@@ -83,22 +83,21 @@ class TcpInputWithGrokTest < Test::Unit::TestCase
 
   def internal_test_grok(conf, tests)
     d = create_driver(BASE_CONFIG + conf)
-    d.run do
+    d.run(expect_emits: tests.size) do
       tests.each {|test|
         TCPSocket.open('127.0.0.1', PORT) do |s|
           s.send(test['msg'], 0)
         end
       }
-      sleep 1
     end
 
-    compare_test_result(d.emits, tests)
+    compare_test_result(d.events, tests)
   end
 
-  def compare_test_result(emits, tests)
-    assert_equal(2, emits.size)
-    emits.each_index {|i|
-      assert_equal(tests[i]['expected'], emits[i][2]['message'])
+  def compare_test_result(events, tests)
+    assert_equal(2, events.size)
+    events.each_index {|i|
+      assert_equal(tests[i]['expected'], events[i][2]['message'])
     }
   end
 end

data/test/test_multiline_grok_parser.rb

@@ -21,7 +21,7 @@ MESSAGE
     ]
     d = create_driver(conf)
 
-    d.parse(text) do |time, record|
+    d.instance.parse(text) do |_time, record|
       assert_equal({ "hostname" => "host1", "message" => message }, record)
     end
   end
@@ -44,7 +44,7 @@ TEXT
       "message2" => "message2",
       "message3" => "message3"
     }
-    d.parse(text) do |time, record|
+    d.instance.parse(text) do |_time, record|
       assert_equal(expected, record)
     end
   end
@@ -63,6 +63,6 @@ TEXT
   private
 
   def create_driver(conf)
-    Fluent::Test::ParserTestDriver.new(TextParser::MultilineGrokParser).configure(conf)
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::MultilineGrokParser).configure(conf)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grok-parser
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 2.0.0
 platform: ruby
 authors:
 - kiyoto
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-26 00:00:00.000000000 Z
+date: 2016-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,20 +58,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.14.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.14.6
 description: 
 email:
 - kiyoto@treasure-data.com
@@ -135,7 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.6.4
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to support Logstash-inspired Grok format for parsing logs