fluent-plugin-grok-parser 2.3.0 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bb787befdf7b8b3d1451c48cff1462e0f3ca40db0bf3c97b3a76ae112e509cdf
-  data.tar.gz: f783cd9143f07e7c5c95cc5da9aeb3a80c20549befa29ca8bcc7814b31fde0ee
+  metadata.gz: 7ee0a491eb3c582a83f228c34648869c693cf028aba0beed0404f0ffa3dff182
+  data.tar.gz: b421584afadea006497e075a5e198bbd2497654cb446148e1223320172a85aac
 SHA512:
-  metadata.gz: 3052cb7600aeeea43ac4c2d7ccf610cfc552e8d88c5e309487d90a843a054ff57d94ae139effa2d46e343a95a095b3c18b42f9d447d4e2b8e644103fd0ad2b6f
-  data.tar.gz: 9b48ea10a3f51661f9343d691271a0ac55a5c971834033401b54ecd6a001507e414bc5240d3d5ccaf4699e8b6d410d2142c697a269abb1fd7d855fea57adfa84
+  metadata.gz: 0db5d145e6d33b285393c75a8383b93120ba623f99859356831e2b08622ca8d1c12ac37d9597ea5fc123d2997c7d42d6b75f4015e9502eaccc830fe41a1919db
+  data.tar.gz: 55d52fd681a761529947a779ebec12de1b7468e1d95546b086fd34d1dcaba6a4122726a954ddbc2f26cd872ca401ec0d6a4ce1294118b8db4bc48a47df79fade

data/appveyor.yml

@@ -1,10 +1,15 @@
 version: '{build}'
+
+# init:
+#   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+
 install:
   - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
-  - "%devkit%\\devkitvars.bat"
+  - IF %ridk%==0 "%devkit%\\devkitvars.bat"
   - ruby --version
   - gem --version
-  - bundle install
+  - IF %ridk%==0 bundle install
+  - IF %ridk%==1 ridk.cmd exec bundle install
 build: off
 test_script:
   - bundle exec rake test
@@ -12,20 +17,34 @@ test_script:
 # https://www.appveyor.com/docs/installed-software/#ruby
 environment:
   matrix:
+    - ruby_version: "25-x64"
+      ridk: 1
+    - ruby_version: "25"
+      ridk: 1
+    - ruby_version: "24-x64"
+      ridk: 1
+    - ruby_version: "24"
+      ridk: 1
     - ruby_version: "23-x64"
       devkit: C:\Ruby23-x64\DevKit
+      ridk: 0
     - ruby_version: "23"
       devkit: C:\Ruby23\DevKit
+      ridk: 0
     - ruby_version: "22-x64"
       devkit: C:\Ruby23-x64\DevKit
+      ridk: 0
     - ruby_version: "21-x64"
       devkit: C:\Ruby23-x64\DevKit
+      ridk: 0
     - ruby_version: "22"
       devkit: C:\Ruby23\DevKit
       WIN_RAPID: true
+      ridk: 0
     - ruby_version: "21"
       devkit: C:\Ruby23\DevKit
       WIN_RAPID: true
+      ridk: 0
 matrix:
   allow_failures:
     - ruby_version: "21"

data/fluent-plugin-grok-parser.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-grok-parser"
-  spec.version       = "2.3.0"
+  spec.version       = "2.4.0"
   spec.authors       = ["kiyoto", "Kenji Okimoto"]
   spec.email         = ["kiyoto@treasure-data.com", "okimoto@clear-code.com"]
   spec.summary       = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}
@@ -19,5 +19,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit", ">=3.1.5"
-  spec.add_runtime_dependency "fluentd", ">=0.14.6"
+  spec.add_runtime_dependency "fluentd", ">=0.14.6", "< 2"
 end

data/lib/fluent/plugin/grok.rb

@@ -36,6 +36,9 @@ module Fluent
       if @plugin.respond_to?(:keep_time_key)
         @keep_time_key = @plugin.keep_time_key
       end
+      if @plugin.respond_to?(:time_format)
+        @time_format = @plugin.time_format
+      end
     end
 
     def add_patterns_from_file(path)
@@ -48,10 +51,10 @@ module Fluent
 
     def setup
       if @plugin.grok_pattern
-        @parsers[:grok_pattern] = expand_pattern_expression(@plugin.grok_pattern, @conf)
+        @parsers[:grok_pattern] = expand_pattern_expression_grok_pattern(@plugin.grok_pattern, @conf)
       else
         @plugin.grok_confs.each.with_index do |grok_conf, index|
-          @parsers[grok_conf.name || index] = expand_pattern_expression(grok_conf.pattern, grok_conf)
+          @parsers[grok_conf.name || index] = expand_pattern_expression_grok_section(grok_conf)
         end
       end
       @parsers.reject! do |key, parser|
@@ -64,7 +67,7 @@ module Fluent
 
     private
 
-    def expand_pattern_expression(grok_pattern, conf)
+    def expand_pattern_expression_grok_pattern(grok_pattern, conf)
       regexp, types = expand_pattern(grok_pattern)
       $log.info "Expanded the pattern #{grok_pattern} into #{regexp}"
       _conf = conf.to_h
@@ -83,6 +86,37 @@ module Fluent
       nil
     end
 
+    def expand_pattern_expression_grok_section(conf)
+      regexp, types = expand_pattern(conf.pattern)
+      $log.info "Expanded the pattern #{conf.pattern} into #{regexp}"
+      _conf = conf.to_h
+      unless types.empty?
+        _conf["types"] = types.map{|subname,type| "#{subname}:#{type}" }.join(",")
+      end
+      if conf["multiline"] ||  @multiline_mode
+        _conf["multiline"] = conf["multiline"] ||  @multiline_mode
+      end
+      if conf["keep_time_key"] || @keep_time_key
+        _conf["keep_time_key"] = conf["keep_time_key"] || @keep_time_key
+      end
+      if conf["time_key"]
+        _conf["time_key"] = conf["time_key"]
+      end
+      if conf["time_format"] || @time_format
+        _conf["time_format"] = conf["time_format"] || @time_format
+      end
+      _conf["expression"] = regexp
+      config = Fluent::Config::Element.new("parse", "", _conf, [])
+      parser = Fluent::Plugin::RegexpParser.new
+      parser.configure(config)
+      parser
+    rescue GrokPatternNotFoundError => e
+      raise e
+    rescue => e
+      $log.error(error: e)
+      nil
+    end
+
     def expand_pattern(pattern)
       # It's okay to modify in place. no need to expand it more than once.
       type_map = {}

data/lib/fluent/plugin/parser_grok.rb

@@ -22,6 +22,9 @@ module Fluent
         config_param :name, :string, default: nil
         desc "The pattern of grok"
         config_param :pattern, :string
+        config_param :keep_time_key, :bool, default: false
+        config_param :time_key, :string, default: "time"
+        config_param :time_format, :string, default: nil
       end
 
       def initialize

data/test/test_grok_parser.rb

@@ -321,6 +321,34 @@ class GrokParserTest < ::Test::Unit::TestCase
     end
   end
 
+  sub_test_case "grok section" do
+    test "complex pattern" do
+      d = create_driver(%[
+        <grok>
+          pattern %{COMBINEDAPACHELOG}
+          time_key timestamp
+          time_format %d/%b/%Y:%H:%M:%S %z
+        </grok>
+      ])
+      expected_record = {
+        "clientip"    => "127.0.0.1",
+        "ident"       => "192.168.0.1",
+        "auth"        => "-",
+        "verb"        => "GET",
+        "request"     => "/",
+        "httpversion" => "1.1",
+        "response"    => "200",
+        "bytes"       => "777",
+        "referrer"    => "\"-\"",
+        "agent"       => "\"Opera/12.0\""
+      }
+      d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') do |time, record|
+        assert_equal(expected_record, record)
+        assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: "%d/%b/%Y:%H:%M:%S %z"), time)
+      end
+    end
+  end
+
   private
 
   def create_driver(conf)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grok-parser
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.4.0
 platform: ruby
 authors:
 - kiyoto
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-10 00:00:00.000000000 Z
+date: 2018-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -60,6 +60,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.6
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -67,6 +70,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.6
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 description: 
 email:
 - kiyoto@treasure-data.com