fluent-plugin-grok-parser 0.0.4 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3677fbf9646ab3f74f7e182c89d65bb8e12c7912
-  data.tar.gz: b922f2cc49ad69e8c39f9c9b3948ad174a327f8e
+  metadata.gz: 939a55ebb2788709efae355b0cecb6109833c05f
+  data.tar.gz: 6b60181db56be7303078d767f0e40b8b980ddcd8
 SHA512:
-  metadata.gz: 5154e8e2ee992db507da6fc4a6dde6aa57882a6bc12e228900793921d8234d1b266c572d5897a5c13fe84b187b6cb044cafffff4c74e10b8dc6836b947db6c68
-  data.tar.gz: 34a14a13f62a00e3c6c148e9668e1a58ac77e0fce1a79f48ef94f8d7d3eb85ee95a581d130d5cdfd9790b4e14b30fe1d3efada3bd6db0f0432b03a4fce86f7bb
+  metadata.gz: 60a0f3d0cf432f1f0202baa92281bf19127105cf259624d670fbdb816b996a8baed8eb68d405767b3676cdb2ee886635815cdd44cba8fdd19a658e7159c1ff54
+  data.tar.gz: ffcb09c3c7c459daabe83de859048d7ac30b7e43fe95bf49594f9cb35b5d45d74cec862d5a61041f61c793cd9e1b93e81b53bd986fe4c94ef25cb0c5d21e32b0

data/README.md

@@ -15,7 +15,7 @@ extracts the first IP address that matches in the log.
 
 ```aconf
 <source>
-  type tail
+  @type tail
   path /path/to/log
   format grok
   grok_pattern %{IP:ip_address}
@@ -27,7 +27,7 @@ extracts the first IP address that matches in the log.
 
 ```aconf
 <source>
-  type tail
+  @type tail
   path /path/to/log
   format grok
   <grok>
@@ -50,7 +50,7 @@ You can parse multiple line text.
 
 ```aconf
 <source>
-  type tail
+  @type tail
   path /path/to/log
   format multiline_grok
   grok_pattern %{IP:ip_address}\n%{GREEDYDATA:message}
@@ -63,7 +63,7 @@ You can use multiple grok patterns to parse your data.
 
 ```aconf
 <source>
-  type tail
+  @type tail
   path /path/to/log
   format multiline_grok
   <grok>
@@ -104,7 +104,7 @@ This is what the `custom_pattern_path` parameter is for.
 
 ```
 <source>
-  type tail
+  @type tail
   path /path/to/log
   format grok
   grok_pattern %{MY_SUPER_PATTERN}
@@ -123,17 +123,15 @@ Although every parsed field has type `string` by default, you can specify other
 The syntax is
 
 ```
-types <field_name_1>:<type_name_1>,<field_name_2>:<type_name_2>,...
+grok_pattern %{GROK_PATTERN:NAME:TYPE}...
 ```
 
 e.g.,
 
 ```
-types user_id:integer,paid:bool,paid_usd_amount:float
+grok_pattern %{INT:foo:integer}
 ```
 
-As demonstrated above, "," is used to delimit field-type pairs while ":" is used to separate a field name with its intended type.
-
 Unspecified fields are parsed at the default string type.
 
 The list of supported types are shown below:
@@ -145,7 +143,7 @@ The list of supported types are shown below:
 * `time`
 * `array`
 
-For the `time` and `array` types, there is an optional third field after the type name. For the "time" type, you can specify a time format like you would in `time_format`.
+For the `time` and `array` types, there is an optional 4th field after the type name. For the "time" type, you can specify a time format like you would in `time_format`.
 
 For the "array" type, the third field specifies the delimiter (the default is ","). For example, if a field called "item\_ids" contains the value "3,4,5", `types item_ids:array` parses it as ["3", "4", "5"]. Alternatively, if the value is "Adam|Alice|Bob", `types item_ids:array:|` parses it as ["Adam", "Alice", "Bob"].
 
@@ -153,11 +151,10 @@ Here is a sample config using the Grok parser with `in_tail` and the `types` par
 
 ```aconf
 <source>
-  type tail
+  @type tail
   path /path/to/log
   format grok
-  grok_pattern %{INT:user_id} paid %{NUMBER:paid_amount}
-  types user_id:integer,paid_amount:float
+  grok_pattern %{INT:user_id:integer} paid %{NUMBER:paid_amount:float}
   tag payment
 </source>
 ```

data/fluent-plugin-grok-parser.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-grok-parser"
-  spec.version       = "0.0.4"
+  spec.version       = "0.1.0"
   spec.authors       = ["kiyoto"]
   spec.email         = ["kiyoto@treasure-data.com"]
   spec.summary       = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}

data/lib/fluent/plugin/grok.rb

@@ -9,7 +9,10 @@ module Fluent
         /%\{    # match '%{' not prefixed with '\'
            (?<name>     # match the pattern name
              (?<pattern>[A-z0-9]+)
-             (?::(?<subname>[@\[\]A-z0-9_:.-]+))?
+             (?::(?<subname>[@\[\]A-z0-9_:.-]+?)
+                  (?::(?<type>(?:string|bool|integer|float|
+                                 time(?::.+)?|
+                                 array(?::.)?)))?)?
            )
          \}/x
 
@@ -51,12 +54,15 @@ module Fluent
     private
 
     def expand_pattern_expression(grok_pattern, conf)
-      regexp = expand_pattern(grok_pattern)
+      regexp, types = expand_pattern(grok_pattern)
       $log.info "Expanded the pattern #{conf['grok_pattern']} into #{regexp}"
       options = nil
       if @multiline_mode
         options = Regexp::MULTILINE
       end
+      unless types.empty?
+        conf["types"] = types.map{|subname,type| "#{subname}:#{type}" }.join(",")
+      end
       TextParser::RegexpParser.new(Regexp.new(regexp, options), conf)
     rescue GrokPatternNotFoundError => e
       raise e
@@ -66,20 +72,22 @@ module Fluent
 
     def expand_pattern(pattern)
       # It's okay to modify in place. no need to expand it more than once.
+      type_map = {}
       while true
         m = PATTERN_RE.match(pattern)
         break unless m
         curr_pattern = @pattern_map[m["pattern"]]
         raise GrokPatternNotFoundError unless curr_pattern
-        replacement_pattern = if m["subname"]
-                                "(?<#{m["subname"]}>#{curr_pattern})"
-                              else
-                                curr_pattern
-                              end
+        if m["subname"]
+          replacement_pattern = "(?<#{m["subname"]}>#{curr_pattern})"
+          type_map[m["subname"]] = m["type"] || "string"
+        else
+          replacement_pattern = curr_pattern
+        end
         pattern.sub!(m[0]) do |s| replacement_pattern end
       end
 
-      pattern
+      [pattern, type_map]
     end
   end
 end

data/lib/fluent/plugin/parser_grok.rb

@@ -6,8 +6,20 @@ module Fluent
 
     class GrokParser < Parser
       Plugin.register_parser('grok', self)
+
+      # For fluentd v0.12.16 or earlier
+      class << self
+        unless method_defined?(:desc)
+          def desc(description)
+          end
+        end
+      end
+
+      desc 'The format of the time field.'
       config_param :time_format, :string, :default => nil
+      desc 'The pattern of grok'
       config_param :grok_pattern, :string, :default => nil
+      desc 'Path to the file that includes custom grok patterns'
       config_param :custom_pattern_path, :string, :default => nil
 
       def initialize

data/lib/fluent/plugin/parser_multiline_grok.rb

@@ -4,8 +4,9 @@ module Fluent
   class TextParser
     class MultilineGrokParser < GrokParser
       Plugin.register_parser('multiline_grok', self)
-      config_param :multiline_start_regexp, :string, :default => nil
 
+      desc 'The regexp to match beginning of multiline'
+      config_param :multiline_start_regexp, :string, :default => nil
 
       def initialize
         super

data/test/test_grok_parser.rb

@@ -15,15 +15,26 @@ def str2time(str_time, format = nil)
 end
 
 class GrokParserTest < ::Test::Unit::TestCase
-  def test_call_for_timestamp
-    internal_test_grok_pattern('%{TIMESTAMP_ISO8601:time}', 'Some stuff at 2014-01-01T00:00:00+0900',
-                               str2time('2014-01-01T00:00:00+0900'), {})
-    internal_test_grok_pattern('%{DATESTAMP_RFC822:time}', 'Some stuff at Mon Aug 15 2005 15:52:01 UTC',
-                               str2time('Mon Aug 15 2005 15:52:01 UTC'), {})
-    internal_test_grok_pattern('%{DATESTAMP_RFC2822:time}', 'Some stuff at Mon, 15 Aug 2005 15:52:01 +0000',
-                               str2time('Mon, 15 Aug 2005 15:52:01 +0000'), {})
-    internal_test_grok_pattern('%{SYSLOGTIMESTAMP:time}', 'Some stuff at Aug 01 00:00:00',
-                               str2time('Aug 01 00:00:00'), {})
+  class Timestamp < self
+    def test_timestamp_iso8601
+      internal_test_grok_pattern('%{TIMESTAMP_ISO8601:time}', 'Some stuff at 2014-01-01T00:00:00+0900',
+                                 str2time('2014-01-01T00:00:00+0900'), {})
+    end
+
+    def test_datestamp_rfc822_with_zone
+      internal_test_grok_pattern('%{DATESTAMP_RFC822:time}', 'Some stuff at Mon Aug 15 2005 15:52:01 UTC',
+                                 str2time('Mon Aug 15 2005 15:52:01 UTC'), {})
+    end
+
+    def test_datestamp_rfc822_with_numeric_zone
+      internal_test_grok_pattern('%{DATESTAMP_RFC2822:time}', 'Some stuff at Mon, 15 Aug 2005 15:52:01 +0000',
+                                 str2time('Mon, 15 Aug 2005 15:52:01 +0000'), {})
+    end
+
+    def test_syslogtimestamp
+      internal_test_grok_pattern('%{SYSLOGTIMESTAMP:time}', 'Some stuff at Aug 01 00:00:00',
+                                 str2time('Aug 01 00:00:00'), {})
+    end
   end
 
   def test_call_for_grok_pattern_not_found
@@ -70,6 +81,51 @@ class GrokParserTest < ::Test::Unit::TestCase
     end
   end
 
+  class OptionalType < self
+    def test_simple
+      internal_test_grok_pattern('%{INT:user_id:integer} paid %{NUMBER:paid_amount:float}',
+                                 '12345 paid 6789.10', nil,
+                                 {"user_id" => 12345, "paid_amount" => 6789.1 })
+    end
+
+    def test_array
+      internal_test_grok_pattern('%{GREEDYDATA:message:array}',
+                                 'a,b,c,d', nil,
+                                 {"message" => %w(a b c d)})
+    end
+
+    def test_array_with_delimiter
+      internal_test_grok_pattern('%{GREEDYDATA:message:array:|}',
+                                 'a|b|c|d', nil,
+                                 {"message" => %w(a b c d)})
+    end
+
+    def test_timestamp_iso8601
+      internal_test_grok_pattern('%{TIMESTAMP_ISO8601:stamp:time}', 'Some stuff at 2014-01-01T00:00:00+0900',
+                                 nil, {"stamp" => str2time('2014-01-01T00:00:00+0900')})
+    end
+
+    def test_datestamp_rfc822_with_zone
+      internal_test_grok_pattern('%{DATESTAMP_RFC822:stamp:time}', 'Some stuff at Mon Aug 15 2005 15:52:01 UTC',
+                                 nil, {"stamp" => str2time('Mon Aug 15 2005 15:52:01 UTC')})
+    end
+
+    def test_datestamp_rfc822_with_numeric_zone
+      internal_test_grok_pattern('%{DATESTAMP_RFC2822:stamp:time}', 'Some stuff at Mon, 15 Aug 2005 15:52:01 +0000',
+                                 nil, {"stamp" => str2time('Mon, 15 Aug 2005 15:52:01 +0000')})
+    end
+
+    def test_syslogtimestamp
+      internal_test_grok_pattern('%{SYSLOGTIMESTAMP:stamp:time}', 'Some stuff at Aug 01 00:00:00',
+                                 nil, {"stamp" => str2time('Aug 01 00:00:00')})
+    end
+
+    def test_timestamp_with_format
+      internal_test_grok_pattern('%{TIMESTAMP_ISO8601:stamp:time:%Y-%m-%d %H%M}', 'Some stuff at 2014-01-01 1000',
+                                 nil, {"stamp" => str2time('2014-01-01 10:00')})
+    end
+  end
+
   private
 
   def internal_test_grok_pattern(grok_pattern, text, expected_time, expected_record, options = {})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grok-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors:
 - kiyoto
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-15 00:00:00.000000000 Z
+date: 2015-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,7 +108,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to support Logstash-inspired Grok format for parsing logs