fluent-plugin-grok-parser 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a517f65e8155156ff9151628ddead36b679d3105
-  data.tar.gz: 7a034a4e0b8d28234ba30612bcf7fd205aa3fbb3
+  metadata.gz: 74360bb48589acb735b21ef7fc33da8e1e8ea1d5
+  data.tar.gz: 6f0b32cff96d5f45fbea166e819132f262c204da
 SHA512:
-  metadata.gz: a16eb5eff0997a710e4dcb7d92ec60ebea27d151d8a85b61c7b7230f4579aeabaf3f54c42d0789f564d579c14eb25c4adbb5c9af0e2527336b9d3e6cb030527b
-  data.tar.gz: 65b1cef569bcf821cdee87dc322088448cc5dfd6fdff08bb4fe9330867e2094a4e848fe660b41d7302b8cc682c537d7d458286083d3566783d04b152c4db5723
+  metadata.gz: af2ee3177353159e23af8d67561294c30b3ac97f7c99ad92e197670689a836c7b029568bf48450782536bc3f59c0a5ab59300a9e4fba11acfb2c29528a669c0c
+  data.tar.gz: 812f3f366210ceda6deec4af94c4da8cc4e401d664cacd6a6e3927b43f55ec30ce6b60be46ef7e8a035f2890c43a59f1b7e54b6492a0d8135a8eab77576368b1

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+
+rvm:
+  - 2.0.0
+  - 2.1
+  - 2.2
+

data/README.md

@@ -44,6 +44,40 @@ extracts the first IP address that matches in the log.
 </source>
 ```
 
+### Multiline support
+
+You can parse multiple line text.
+
+```aconf
+<source>
+  type tail
+  path /path/to/log
+  format multiline_grok
+  grok_pattern %{IP:ip_address}\n%{GREEDYDATA:message}
+  multiline_start_regex /^\s/
+  tag grokked_log
+</source>
+```
+
+You can use multiple grok patterns to parse your data.
+
+```aconf
+<source>
+  type tail
+  path /path/to/log
+  format multiline_grok
+  <grok>
+    pattern Started %{WORD:verb} "%{URIPATH:pathinfo}" for %{IP:ip} at %{TIMESTAMP_ISO8601:timestamp}\nProcessing by %{WORD:controller}#%{WORD:action} as %{WORD:format}%{DATA:message}Completed %{NUMBER:response} %{WORD} in %{NUMBER:elapsed} (%{DATA:elapsed_details})
+  </grok>
+  tag grokked_log
+</source>
+```
+
+Fluentd accumulates data in the buffer forever to parse complete data when no pattern matches.
+
+You can use this parser without `multiline_start_regex` when you know your data structure perfectly.
+
+
 ## How to write Grok patterns
 
 Grok patterns look like `%{PATTERN_NAME:name}` where ":name" is optional. If "name" is provided, then it
@@ -80,6 +114,54 @@ This is what the `custom_pattern_path` parameter is for.
 
 `custom_pattern_path` can be either a directory or file. If it's a directory, it reads all the files in it.
 
+## FAQs
+
+### 1. How can I convert types of the matched patterns like Logstash's Grok?
+
+Although every parsed field has type `string` by default, you can specify other types. This is useful when filtering particular fields numerically or storing data with sensible type information.
+
+The syntax is
+
+```
+types <field_name_1>:<type_name_1>,<field_name_2>:<type_name_2>,...
+```
+
+e.g.,
+
+```
+types user_id:integer,paid:bool,paid_usd_amount:float
+```
+
+As demonstrated above, "," is used to delimit field-type pairs while ":" is used to separate a field name with its intended type.
+
+Unspecified fields are parsed at the default string type.
+
+The list of supported types are shown below:
+
+* `string`
+* `bool`
+* `integer` ("int" would NOT work!)
+* `float`
+* `time`
+* `array`
+
+For the `time` and `array` types, there is an optional third field after the type name. For the "time" type, you can specify a time format like you would in `time_format`.
+
+For the "array" type, the third field specifies the delimiter (the default is ","). For example, if a field called "item\_ids" contains the value "3,4,5", `types item_ids:array` parses it as ["3", "4", "5"]. Alternatively, if the value is "Adam|Alice|Bob", `types item_ids:array:|` parses it as ["Adam", "Alice", "Bob"].
+
+Here is a sample config using the Grok parser with `in_tail` and the `types` parameter:
+
+```aconf
+<source>
+  type tail
+  path /path/to/log
+  format grok
+  grok_pattern %{INT:user_id} paid %{NUMBER:paid_amount}
+  types user_id:integer,paid_amount:float
+  tag payment
+</source>
+```
+
 ## License
 
 Apache 2.0 License

data/fluent-plugin-grok-parser.gemspec

@@ -4,12 +4,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-grok-parser"
-  spec.version       = "0.0.2"
+  spec.version       = "0.0.3"
   spec.authors       = ["kiyoto"]
   spec.email         = ["kiyoto@treasure-data.com"]
-  spec.summary       = %q{Fluentd plugin to suppor Logstash-inspired Grok format for parsing logs}
+  spec.summary       = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}
   spec.homepage      = "https://github.com/kiyoto/fluent-plugin-grok-parser"
-  spec.license       = "Apache License, Version 2.0"
+  spec.license       = "Apache-2.0"
 
   spec.files         = `git ls-files`.split($/)
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
@@ -18,5 +18,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "test-unit", ">=3.1.5"
   spec.add_runtime_dependency "fluentd", ">=0.10.58"
 end

data/lib/fluent/plugin/grok.rb

@@ -0,0 +1,84 @@
+module Fluent
+  class Grok
+    class GrokPatternNotFoundError < StandardError
+    end
+
+    # Much of the Grok implementation is based on Jordan Sissel's jls-grok
+    # See https://github.com/jordansissel/ruby-grok/blob/master/lib/grok-pure.rb
+    PATTERN_RE = \
+        /%\{    # match '%{' not prefixed with '\'
+           (?<name>     # match the pattern name
+             (?<pattern>[A-z0-9]+)
+             (?::(?<subname>[@\[\]A-z0-9_:.-]+))?
+           )
+         \}/x
+
+    attr_reader :parsers
+
+    def initialize(plugin, conf)
+      @pattern_map = {}
+      @parsers = []
+      @multiline_mode = false
+      @conf = conf
+      if plugin.instance_of?(Fluent::TextParser::MultilineGrokParser)
+        @multiline_mode = true
+      end
+      if @conf['multiline_start_regexp']
+        @multiline_start_regexp = Regexp.compile(@conf['multiline_start_regexp'][1..-2])
+      end
+    end
+
+    def add_patterns_from_file(path)
+      File.new(path).each_line do |line|
+        next if line[0] == '#' || /^$/ =~ line
+        name, pat = line.chomp.split(/\s+/, 2)
+        @pattern_map[name] = pat
+      end
+    end
+
+    def setup
+      if @conf['grok_pattern']
+        @parsers << expand_pattern_expression(@conf['grok_pattern'], @conf)
+      else
+        grok_confs = @conf.elements.select {|e| e.name == 'grok'}
+        grok_confs.each do |grok_conf|
+          @parsers << expand_pattern_expression(grok_conf['pattern'], grok_conf)
+        end
+      end
+    end
+
+    private
+
+    def expand_pattern_expression(grok_pattern, conf)
+      regexp = expand_pattern(grok_pattern)
+      $log.info "Expanded the pattern #{conf['grok_pattern']} into #{regexp}"
+      options = nil
+      if @multiline_mode
+        options = Regexp::MULTILINE
+      end
+      TextParser::RegexpParser.new(Regexp.new(regexp, options), conf)
+    rescue GrokPatternNotFoundError => e
+      raise e
+    rescue => e
+      $log.error e.backtrace.join("\n")
+    end
+
+    def expand_pattern(pattern)
+      # It's okay to modify in place. no need to expand it more than once.
+      while true
+        m = PATTERN_RE.match(pattern)
+        break unless m
+        curr_pattern = @pattern_map[m["pattern"]]
+        raise GrokPatternNotFoundError unless curr_pattern
+        replacement_pattern = if m["subname"]
+                                "(?<#{m["subname"]}>#{curr_pattern})"
+                              else
+                                curr_pattern
+                              end
+        pattern.sub!(m[0]) do |s| replacement_pattern end
+      end
+
+      pattern
+    end
+  end
+end

data/lib/fluent/plugin/parser_grok.rb

@@ -1,3 +1,5 @@
+require "fluent/plugin/grok"
+
 module Fluent
   class TextParser
     class GrokPatternNotFoundError < Exception; end
@@ -8,87 +10,37 @@ module Fluent
       config_param :grok_pattern, :string, :default => nil
       config_param :custom_pattern_path, :string, :default => nil
 
-      # Much of the Grok implementation is based on Jordan Sissel's jls-grok
-      # See https://github.com/jordansissel/ruby-grok/blob/master/lib/grok-pure.rb
-      PATTERN_RE = \
-          /%\{    # match '%{' not prefixed with '\'
-             (?<name>     # match the pattern name
-               (?<pattern>[A-z0-9]+)
-               (?::(?<subname>[@\[\]A-z0-9_:.-]+))?
-             )
-           \}/x
-
       def initialize
         super
-        @pattern_map = {}
-        default_pattern_dir = File.expand_path('../../../../patterns/*', __FILE__)
-        Dir.glob(default_pattern_dir) do |pattern_file_path|
-          add_patterns_from_file(pattern_file_path)
-        end
         @default_parser = NoneParser.new
-        @parsers = []
       end
 
       def configure(conf={})
         super
 
+        @grok = Grok.new(self, conf)
+
+        default_pattern_dir = File.expand_path('../../../../patterns/*', __FILE__)
+        Dir.glob(default_pattern_dir) do |pattern_file_path|
+          @grok.add_patterns_from_file(pattern_file_path)
+        end
+
         if @custom_pattern_path
           if Dir.exists? @custom_pattern_path
             Dir.glob(@custom_pattern_path + '/*') do |pattern_file_path|
-              add_patterns_from_file(pattern_file_path)
+              @grok.add_patterns_from_file(pattern_file_path)
             end
           elsif File.exists? @custom_pattern_path
-            add_patterns_from_file(@custom_pattern_path)
+            @grok.add_patterns_from_file(@custom_pattern_path)
           end
         end
 
-        if @grok_pattern
-          @parsers = [expand_pattern_exn(@grok_pattern, conf)]
-        else
-          grok_confs = conf.elements.select {|e| e.name == 'grok'}
-          grok_confs.each do |grok_conf|
-            @parsers << expand_pattern_exn(grok_conf['pattern'], grok_conf)
-          end
-        end
-      end
-
-      def add_patterns_from_file(path)
-        File.new(path).each_line do |line|
-          next if line[0] == '#' || /^$/ =~ line
-          name, pat = line.chomp.split(/\s+/, 2)
-          @pattern_map[name] = pat
-        end
-      end
-
-      def expand_pattern_exn(pattern, conf)
-        regexp = expand_pattern(pattern)
-        $log.info "Expanded the pattern #{conf['grok_pattern']} into #{regexp}"
-        RegexpParser.new(Regexp.new(regexp), conf)
-      rescue => e
-        $log.error e.backtrace.join("\n")
-      end
-
-      def expand_pattern(pattern)
-        # It's okay to modify in place. no need to expand it more than once.
-        while true
-          m = PATTERN_RE.match(pattern)
-          break if not m
-          curr_pattern = @pattern_map[m["pattern"]]
-          raise GrokPatternNotFoundError if not curr_pattern
-          replacement_pattern = if m["subname"]
-                                  "(?<#{m["subname"]}>#{curr_pattern})"
-                                else
-                                  curr_pattern
-                                end
-          pattern.sub!(m[0]) do |s| replacement_pattern end
-        end
-      
-        pattern 
+        @grok.setup
       end
 
-      def parse(text)
+      def parse(text, &block)
         if block_given?
-          @parsers.each do |parser|
+          @grok.parsers.each do |parser|
             parser.parse(text) do |time, record|
               if time and record
                 yield time, record
@@ -98,7 +50,7 @@ module Fluent
           end
           yield @default_parser.parse(text)
         else
-          @parsers.each do |parser|
+          @grok.parsers.each do |parser|
             parser.parse(text) do |time, record|
               if time and record
                 return time, record

data/lib/fluent/plugin/parser_multiline_grok.rb

@@ -0,0 +1,48 @@
+require 'fluent/plugin/parser_grok'
+
+module Fluent
+  class TextParser
+    class MultilineGrokParser < GrokParser
+      Plugin.register_parser('multiline_grok', self)
+      config_param :multiline_start_regexp, :string, :default => nil
+
+
+      def initialize
+        super
+      end
+
+      def configure(conf={})
+        super
+      end
+
+      def has_firstline?
+        !!@multiline_start_regexp
+      end
+
+      def firstline?(text)
+        @multiline_start_regexp && !@multiline_start_regexp.match(text)
+      end
+
+      def parse(text, &block)
+        if block_given?
+          @grok.parsers.each do |parser|
+            parser.parse(text) do |time, record|
+              if time and record
+                yield time, record
+                return
+              end
+            end
+          end
+        else
+          @grok.parsers.each do |parser|
+            parser.parse(text) do |time, record|
+              if time and record
+                return time, record
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+
+def unused_port
+  s = TCPServer.open(0)
+  port = s.addr[1]
+  s.close
+  port
+end
+
+def ipv6_enabled?
+  require 'socket'
+
+  begin
+    TCPServer.open("::1", 0)
+    true
+  rescue
+    false
+  end
+end

data/test/test_grok_parser.rb

@@ -15,22 +15,6 @@ def str2time(str_time, format = nil)
 end
 
 class GrokParserTest < ::Test::Unit::TestCase
-  def internal_test_grok_pattern(grok_pattern, text, expected_time, expected_record, options = {})
-    parser = TextParser::GrokParser.new
-    parser.configure({"grok_pattern" => grok_pattern}.merge(options))
-
-    # for the old, return based API
-    time, record = parser.parse(text)
-    assert_equal(expected_time, time) if expected_time
-    assert_equal(expected_record, record)
-
-    # for the new API
-    parser.parse(text) {|time, record|
-      assert_equal(expected_time, time) if expected_time
-      assert_equal(expected_record, record)
-    }
-  end
-
   def test_call_for_timestamp
     internal_test_grok_pattern('%{TIMESTAMP_ISO8601:time}', 'Some stuff at 2014-01-01T00:00:00+0900',
                                str2time('2014-01-01T00:00:00+0900'), {})
@@ -43,7 +27,7 @@ class GrokParserTest < ::Test::Unit::TestCase
   end
 
   def test_call_for_grok_pattern_not_found
-    assert_raise TextParser::GrokPatternNotFoundError do
+    assert_raise Grok::GrokPatternNotFoundError do
       internal_test_grok_pattern('%{THIS_PATTERN_DOESNT_EXIST}', 'Some stuff at somewhere', nil, {})
     end
   end
@@ -85,4 +69,22 @@ class GrokParserTest < ::Test::Unit::TestCase
       File.delete(pattern_file.path)
     end
   end
+
+  private
+
+  def internal_test_grok_pattern(grok_pattern, text, expected_time, expected_record, options = {})
+    parser = TextParser::GrokParser.new
+    parser.configure(Config::Element.new('ROOT', '', {"grok_pattern" => grok_pattern}.merge(options), []))
+
+    # for the old, return based API
+    time, record = parser.parse(text)
+    assert_equal(expected_time, time) if expected_time
+    assert_equal(expected_record, record)
+
+    # for the new API
+    parser.parse(text) {|time, record|
+      assert_equal(expected_time, time) if expected_time
+      assert_equal(expected_record, record)
+    }
+  end
 end

data/test/test_multiline_grok_parser.rb

@@ -0,0 +1,60 @@
+require 'fluent/plugin/parser_multiline_grok'
+require 'fluent/config/parser'
+
+require 'stringio'
+
+class MultilineGrokParserTest < Test::Unit::TestCase
+  def test_multiline
+    text=<<TEXT.chomp
+host1 message1
+ message2
+ message3
+TEXT
+    message =<<MESSAGE.chomp
+message1
+ message2
+ message3
+MESSAGE
+    conf = %[
+      grok_pattern %{HOSTNAME:hostname} %{GREEDYDATA:message}
+      multiline_start_regexp /^\s/
+    ]
+    parser = create_parser(conf)
+
+    parser.parse(text) do |time, record|
+      assert_equal({ "hostname" => "host1", "message" => message }, record)
+    end
+  end
+
+  def test_without_multiline_start_regexp
+    text = <<TEXT.chomp
+host1 message1
+ message2
+ message3
+end
+TEXT
+    conf = %[
+       grok_pattern %{HOSTNAME:hostname} %{DATA:message1}\\n %{DATA:message2}\\n %{DATA:message3}\\nend
+    ]
+    parser = create_parser(conf)
+
+    expected = {
+      "hostname" => "host1",
+      "message1" => "message1",
+      "message2" => "message2",
+      "message3" => "message3"
+    }
+    parser.parse(text) do |time, record|
+      assert_equal(expected, record)
+    end
+  end
+
+  private
+
+  def create_parser(conf)
+    parser = TextParser::MultilineGrokParser.new
+    io = StringIO.new(conf)
+    parser.configure(Config::Parser.parse(io, "fluent.conf"))
+    parser
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grok-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - kiyoto
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-22 00:00:00.000000000 Z
+date: 2015-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.5
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -59,20 +73,24 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".travis.yml"
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
 - fluent-plugin-grok-parser.gemspec
+- lib/fluent/plugin/grok.rb
 - lib/fluent/plugin/parser_grok.rb
+- lib/fluent/plugin/parser_multiline_grok.rb
 - patterns/grok-patterns
 - patterns/nagios
+- test/helper.rb
 - test/test_grok_parser.rb
 - test/test_grok_parser_in_tcp.rb
+- test/test_multiline_grok_parser.rb
 homepage: https://github.com/kiyoto/fluent-plugin-grok-parser
 licenses:
-- Apache License, Version 2.0
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -90,10 +108,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
-summary: Fluentd plugin to suppor Logstash-inspired Grok format for parsing logs
+summary: Fluentd plugin to support Logstash-inspired Grok format for parsing logs
 test_files:
+- test/helper.rb
 - test/test_grok_parser.rb
 - test/test_grok_parser_in_tcp.rb
+- test/test_multiline_grok_parser.rb

data/Gemfile.lock

@@ -1,40 +0,0 @@
-PATH
-  remote: .
-  specs:
-    fluent-plugin-grok-parser (0.0.2)
-      fluentd (>= 0.10.58)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    cool.io (1.2.4)
-    fluentd (0.12.1)
-      cool.io (>= 1.2.2, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
-      json (>= 1.4.3)
-      msgpack (>= 0.5.4, < 0.6.0)
-      sigdump (~> 0.2.2)
-      string-scrub (>= 0.0.3)
-      tzinfo (>= 1.0.0)
-      tzinfo-data (>= 1.0.0)
-      yajl-ruby (~> 1.0)
-    http_parser.rb (0.6.0)
-    json (1.8.1)
-    msgpack (0.5.9)
-    rake (10.1.1)
-    sigdump (0.2.2)
-    string-scrub (0.0.5)
-    thread_safe (0.3.4)
-    tzinfo (1.2.2)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2014.10)
-      tzinfo (>= 1.0.0)
-    yajl-ruby (1.2.1)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler
-  fluent-plugin-grok-parser!
-  rake