fluent-plugin-grok-parser 2.4.0 → 2.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7ee0a491eb3c582a83f228c34648869c693cf028aba0beed0404f0ffa3dff182
-  data.tar.gz: b421584afadea006497e075a5e198bbd2497654cb446148e1223320172a85aac
+  metadata.gz: 9a5f5cd1dd1aa1de7edd4425755616b0488492d32847b8e254d79d0385d3cbd4
+  data.tar.gz: cc629c894253715304a18b3eccc2a2caba04069b9b22124893a8e77dacbf4c44
 SHA512:
-  metadata.gz: 0db5d145e6d33b285393c75a8383b93120ba623f99859356831e2b08622ca8d1c12ac37d9597ea5fc123d2997c7d42d6b75f4015e9502eaccc830fe41a1919db
-  data.tar.gz: 55d52fd681a761529947a779ebec12de1b7468e1d95546b086fd34d1dcaba6a4122726a954ddbc2f26cd872ca401ec0d6a4ce1294118b8db4bc48a47df79fade
+  metadata.gz: 7548442e6f18c3b302df37f9f8d9300374f116236fcecb30c8e2426e65f43038e011a312ff4641d779c26fd95f6f3d38db17145ea793a2a29f34b2ab7c6d0709
+  data.tar.gz: b5e05fbefb609cf206ddbbf8347a9e9f93e4ba4a4eb2aa1d653db6b54332c311b4c388d8cf7e85fad2d10919d444a2bb6a5fe7677024f999d43cfd7680785b9c

data/.travis.yml

@@ -2,8 +2,6 @@ sudo: false
 language: ruby
 
 rvm:
-  - 2.2.10
-  - 2.3.7
-  - 2.4.4
-  - 2.5.1
-
+  - 2.4
+  - 2.5.3
+  - 2.6.0

data/README.md

@@ -95,8 +95,7 @@ You can use this parser without `multiline_start_regexp` when you know your data
 
 ## Configurations
 
-* See also: [TimeParameters Plugin Overview](https://docs.fluentd.org/v1.0/articles/timeparameters-plugin-overview)
-* See also: [Parser Plugin Overview](https://docs.fluentd.org/v1.0/articles/parser-plugin-overview)
+* See also: [Config: Parse Section - Fluentd](https://docs.fluentd.org/configuration/parse-section)
 
 * **time_format** (string) (optional): The format of the time field.
 * **grok_pattern** (string) (optional): The pattern of grok. You cannot specify multiple grok pattern with this.
@@ -105,6 +104,17 @@ You can use this parser without `multiline_start_regexp` when you know your data
 * **grok_name_key** (string) (optional): The key name to store grok section's name
 * **multi_line_start_regexp** (string) (optional): The regexp to match beginning of multiline. This is only for "multiline_grok".
 
+### \<grok\> section (optional) (multiple)
+
+* **name** (string) (optional): The name of this grok section
+* **pattern** (string) (required): The pattern of grok
+* **keep_time_key** (bool) (optional): If true, keep time field in the record.
+* **time_key** (string) (optional): Specify time field for event time. If the event doesn't have this field, current time is used.
+  * Default value: `time`.
+* **time_format** (string) (optional): Process value using specified format. This is available only when time_type is string
+* **timezone** (string) (optional): Use specified timezone. one can parse/format the time value in the specified timezone.
+
+
 ## Examples
 
 ### Using grok\_failure\_key
@@ -184,6 +194,28 @@ This will add keys like following:
 Add `grokfailure` key to the record if the record does not match any grok pattern.
 See also test code for more details.
 
+## How to parse time value using specific timezone
+
+```aconf
+<source>
+  @type tail
+  path /path/to/log
+  tag grokked_log
+  <parse>
+    @type grok
+    <grok>
+      name mylog-without-timezone
+      pattern %{DATESTAMP:time} %{GREEDYDATE:message}
+      timezone Asia/Tokyo
+    </grok>
+  </parse>
+</source>
+```
+
+This will parse the `time` value as "Asia/Tokyo" timezone.
+
+See [Config: Parse Section - Fluentd](https://docs.fluentd.org/configuration/parse-section) for more details about timezone.
+
 ## How to write Grok patterns
 
 Grok patterns look like `%{PATTERN_NAME:name}` where ":name" is optional. If "name" is provided, then it
@@ -271,7 +303,7 @@ Here is a sample config using the Grok parser with `in_tail` and the `types` par
 
 If you want to use this plugin with Fluentd v0.12.x or earlier, you can use this plugin version v1.x.
 
-See also: [Plugin Management | Fluentd](http://docs.fluentd.org/articles/plugin-management#plugin-version-management)
+See also: [Plugin Management | Fluentd](https://docs.fluentd.org/deployment/plugin-management)
 
 ## License
 

data/appveyor.yml

@@ -5,11 +5,9 @@ version: '{build}'
 
 install:
   - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
-  - IF %ridk%==0 "%devkit%\\devkitvars.bat"
   - ruby --version
   - gem --version
-  - IF %ridk%==0 bundle install
-  - IF %ridk%==1 ridk.cmd exec bundle install
+  - ridk.cmd exec bundle install
 build: off
 test_script:
   - bundle exec rake test
@@ -17,34 +15,9 @@ test_script:
 # https://www.appveyor.com/docs/installed-software/#ruby
 environment:
   matrix:
+    - ruby_version: "26-x64"
+    - ruby_version: "26"
     - ruby_version: "25-x64"
-      ridk: 1
     - ruby_version: "25"
-      ridk: 1
     - ruby_version: "24-x64"
-      ridk: 1
     - ruby_version: "24"
-      ridk: 1
-    - ruby_version: "23-x64"
-      devkit: C:\Ruby23-x64\DevKit
-      ridk: 0
-    - ruby_version: "23"
-      devkit: C:\Ruby23\DevKit
-      ridk: 0
-    - ruby_version: "22-x64"
-      devkit: C:\Ruby23-x64\DevKit
-      ridk: 0
-    - ruby_version: "21-x64"
-      devkit: C:\Ruby23-x64\DevKit
-      ridk: 0
-    - ruby_version: "22"
-      devkit: C:\Ruby23\DevKit
-      WIN_RAPID: true
-      ridk: 0
-    - ruby_version: "21"
-      devkit: C:\Ruby23\DevKit
-      WIN_RAPID: true
-      ridk: 0
-matrix:
-  allow_failures:
-    - ruby_version: "21"

data/fluent-plugin-grok-parser.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-grok-parser"
-  spec.version       = "2.4.0"
+  spec.version       = "2.6.2"
   spec.authors       = ["kiyoto", "Kenji Okimoto"]
   spec.email         = ["kiyoto@treasure-data.com", "okimoto@clear-code.com"]
   spec.summary       = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}

data/lib/fluent/plugin/grok.rb

@@ -13,7 +13,7 @@ module Fluent
              (?<pattern>[A-z0-9]+)
              (?::(?<subname>[@\[\]A-z0-9_:.-]+?)
                   (?::(?<type>(?:string|bool|integer|float|
-                                 time(?::.+)?|
+                                 time(?::.+?)?|
                                  array(?::.)?)))?)?
            )
          \}/x
@@ -27,6 +27,8 @@ module Fluent
       @multiline_mode = false
       @conf = conf
       @plugin = plugin
+      @time_format = nil
+      @timezone = nil
       if @plugin.respond_to?(:firstline?)
         @multiline_mode = true
       end
@@ -39,6 +41,9 @@ module Fluent
       if @plugin.respond_to?(:time_format)
         @time_format = @plugin.time_format
       end
+      if @plugin.respond_to?(:timezone)
+        @timezone = @plugin.timezone
+      end
     end
 
     def add_patterns_from_file(path)
@@ -105,6 +110,9 @@ module Fluent
       if conf["time_format"] || @time_format
         _conf["time_format"] = conf["time_format"] || @time_format
       end
+      if conf["timezone"] || @timezone
+        _conf["timezone"] = conf["timezone"] || @timezone
+      end
       _conf["expression"] = regexp
       config = Fluent::Config::Element.new("parse", "", _conf, [])
       parser = Fluent::Plugin::RegexpParser.new
@@ -131,7 +139,7 @@ module Fluent
         else
           replacement_pattern = "(?:#{curr_pattern})"
         end
-        pattern.sub!(m[0]) do |s|
+        pattern = pattern.sub(m[0]) do |s|
           replacement_pattern
         end
       end

data/lib/fluent/plugin/parser_grok.rb

@@ -22,9 +22,14 @@ module Fluent
         config_param :name, :string, default: nil
         desc "The pattern of grok"
         config_param :pattern, :string
+        desc "If true, keep time field in the record."
         config_param :keep_time_key, :bool, default: false
+        desc "Specify time field for event time. If the event doesn't have this field, current time is used."
         config_param :time_key, :string, default: "time"
+        desc "Process value using specified format. This is available only when time_type is string"
         config_param :time_format, :string, default: nil
+        desc "Use specified timezone. one can parse/format the time value in the specified timezone."
+        config_param :timezone, :string, default: nil
       end
 
       def initialize

data/patterns/grok-patterns

@@ -37,7 +37,7 @@ PATH (?:%{UNIXPATH}|%{WINPATH})
 UNIXPATH (/([\w_%!$@:.,+~-]+|\\.)*)+
 TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
 WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
-URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
+URIPROTO [A-Za-z]([A-Za-z0-9+\-.]+)+
 URIHOST %{IPORHOST}(?::%{POSINT:port})?
 # uripath comes loosely from RFC1738, but mostly from what Firefox
 # doesn't turn into %XX
@@ -48,7 +48,7 @@ URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
 URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
 
 # Months: January, Feb, 3, 03, 12, December
-MONTH \b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\b
+MONTH \b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y|i)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\b
 MONTHNUM (?:0?[1-9]|1[0-2])
 MONTHNUM2 (?:0[1-9]|1[0-2])
 MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
@@ -92,4 +92,4 @@ QS %{QUOTEDSTRING}
 SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
 
 # Log Levels
-LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
+LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo?(?:rmation)?|INFO?(?:RMATION)?|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)

data/patterns/haproxy

@@ -31,7 +31,7 @@ HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:captured_response_headers}
 # HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:response_header_content_type}\|%{DATA:response_header_content_encoding}\|%{DATA:response_header_cache_control}\|%{DATA:response_header_last_modified}
 
 # parse a haproxy 'httplog' line
-HAPROXYHTTPBASE %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?"
+HAPROXYHTTPBASE %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?"?
 
 HAPROXYHTTP (?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{HAPROXYHTTPBASE}
 

data/patterns/httpd

@@ -2,14 +2,14 @@ HTTPDUSER %{EMAILADDRESS}|%{USER}
 HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
 
 # Log formats
-HTTPD_COMMONLOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{HTTPDUSER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
+HTTPD_COMMONLOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{HTTPDUSER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" (?:-|%{NUMBER:response}) (?:-|%{NUMBER:bytes})
 HTTPD_COMBINEDLOG %{HTTPD_COMMONLOG} %{QS:referrer} %{QS:agent}
 
 # Error logs
 HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:message}
-HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_message}:)?( \[client %{IPORHOST:clientip}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
+HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}(:tid %{NUMBER:tid})?\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_message}:)?( \[client %{IPORHOST:clientip}:%{POSINT:clientport}\])?( %{DATA:errorcode}:)? %{GREEDYDATA:message}
 HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
 
 # Deprecated
 COMMONAPACHELOG %{HTTPD_COMMONLOG}
-COMBINEDAPACHELOG %{HTTPD_COMBINEDLOG}
+COMBINEDAPACHELOG %{HTTPD_COMBINEDLOG}

data/patterns/java

@@ -1,14 +1,12 @@
 JAVACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]*
 #Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source'
-JAVAFILE (?:[A-Za-z0-9_. -]+)
+JAVAFILE (?:[a-zA-Z$_0-9. -]+)
 #Allow special <init>, <clinit> methods
 JAVAMETHOD (?:(<(?:cl)?init>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
 #Line number is optional in special cases 'Native method' or 'Unknown source'
 JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\)
 # Java Logs
 JAVATHREAD (?:[A-Z]{2}-Processor[\d]+)
-JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+
-JAVAFILE (?:[A-Za-z0-9_.-]+)
 JAVALOGMESSAGE (.*)
 # MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM
 CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)

data/patterns/linux-syslog

@@ -11,6 +11,6 @@ SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message}
 # IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424)
 SYSLOG5424PRI <%{NONNEGINT:syslog5424_pri}>
 SYSLOG5424SD \[%{DATA}\]+
-SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)
+SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{IPORHOST:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)
 
 SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg}

data/patterns/nagios

@@ -89,7 +89,7 @@ NAGIOS_PASSIVE_HOST_CHECK %{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios_type}: %{DATA:
 NAGIOS_SERVICE_EVENT_HANDLER %{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}
 NAGIOS_HOST_EVENT_HANDLER %{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name}
 
-NAGIOS_TIMEPERIOD_TRANSITION %{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{DATA:nagios_unknown1};%{DATA:nagios_unknown2}
+NAGIOS_TIMEPERIOD_TRANSITION %{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{NUMBER:nagios_unknown1};%{NUMBER:nagios_unknown2}
 
 ####################
 #### External checks

data/test/test_grok_parser.rb

@@ -158,37 +158,49 @@ class GrokParserTest < ::Test::Unit::TestCase
     end
   end
 
-  test "no grok patterns" do
-    assert_raise Fluent::ConfigError do
-      create_driver('')
+  sub_test_case "configure" do
+    test "no grok patterns" do
+      assert_raise Fluent::ConfigError do
+        create_driver('')
+      end
+    end
+
+    test "invalid config value type" do
+      assert_raise Fluent::ConfigError do
+        create_driver(%[
+          <grok>
+            pattern %{PATH:path:foo}
+          </grok>
+        ])
+      end
     end
-  end
 
-  test "invalid config value type" do
-    assert_raise Fluent::ConfigError do
-      create_driver(%[
+    test "invalid config value type and normal grok pattern" do
+      d = create_driver(%[
         <grok>
           pattern %{PATH:path:foo}
         </grok>
+        <grok>
+          pattern %{IP:ip_address}
+        </grok>
       ])
+      assert_equal(1, d.instance.instance_variable_get(:@grok).parsers.size)
+      logs = $log.instance_variable_get(:@logger).instance_variable_get(:@logdev).logs
+      error_logs = logs.grep(/error_class/)
+      assert_equal(1, error_logs.size)
+      error_message = error_logs.first[/error="(.+)"/, 1]
+      assert_equal("unknown value conversion for key:'path', type:'foo'", error_message)
     end
-  end
 
-  test "invalid config value type and normal grok pattern" do
-    d = create_driver(%[
-      <grok>
-        pattern %{PATH:path:foo}
-      </grok>
-      <grok>
-        pattern %{IP:ip_address}
-      </grok>
-    ])
-    assert_equal(1, d.instance.instance_variable_get(:@grok).parsers.size)
-    logs = $log.instance_variable_get(:@logger).instance_variable_get(:@logdev).logs
-    error_logs = logs.grep(/error_class/)
-    assert_equal(1, error_logs.size)
-    error_message = error_logs.first[/error="(.+)"/, 1]
-    assert_equal("unknown value conversion for key:'path', type:'foo'", error_message)
+    test "keep original configuration" do
+      config = %[
+        <grok>
+          pattern %{INT:user_id:integer} paid %{NUMBER:paid_amount:float}
+        </grok>
+      ]
+      d = create_driver(config)
+      assert_equal("%{INT:user_id:integer} paid %{NUMBER:paid_amount:float}", d.instance.config.elements("grok").first["pattern"])
+    end
   end
 
   sub_test_case "grok_name_key" do
@@ -347,6 +359,61 @@ class GrokParserTest < ::Test::Unit::TestCase
         assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: "%d/%b/%Y:%H:%M:%S %z"), time)
       end
     end
+
+    test "leading time type with following other type" do
+      d = create_driver(%[
+        <grok>
+          pattern \\[%{HTTPDATE:log_timestamp:time:%d/%b/%Y:%H:%M:%S %z}\\] %{GREEDYDATA:message}
+        </grok>
+      ])
+      expected_record = {
+        "log_timestamp" => event_time("03/Feb/2019:06:47:21 +0530", format: "%d/%b/%Y:%H:%M:%S %z"),
+        "message" => "Python-urllib/2.7"
+      }
+      d.instance.parse('[03/Feb/2019:06:47:21 +0530] Python-urllib/2.7') do |time, record|
+        assert_equal(expected_record, record)
+      end
+    end
+
+    test "timezone" do
+      d = create_driver(%[
+        <grok>
+          pattern %{TIMESTAMP_ISO8601:time} %{GREEDYDATA:message}
+          time_key time
+          time_format %Y-%m-%d %H:%M:%S
+          timezone Europe/Berlin
+        </grok>
+      ])
+      d.instance.parse("2019-02-01 12:34:56 This is test") do |time, record|
+        assert_equal(event_time("2019-02-01 12:34:56 +0100"), time)
+        assert_equal({ "message" => "This is test" }, record)
+      end
+    end
+
+    test "multiple timezone" do
+      d = create_driver(%[
+        <grok>
+          pattern %{TIMESTAMP_ISO8601:time} 1 %{GREEDYDATA:message}
+          time_key time
+          time_format %Y-%m-%d %H:%M:%S
+          timezone Europe/Berlin
+        </grok>
+        <grok>
+          pattern %{TIMESTAMP_ISO8601:time} 2 %{GREEDYDATA:message}
+          time_key time
+          time_format %Y-%m-%d %H:%M:%S
+          timezone Asia/Aden
+        </grok>
+      ])
+      d.instance.parse("2019-02-01 12:34:56 1 This is test") do |time, record|
+        assert_equal(event_time("2019-02-01 12:34:56 +0100"), time)
+        assert_equal({ "message" => "This is test" }, record)
+      end
+      d.instance.parse("2019-02-01 12:34:56 2 This is test") do |time, record|
+        assert_equal(event_time("2019-02-01 12:34:56 +0300"), time)
+        assert_equal({ "message" => "This is test" }, record)
+      end
+    end
   end
 
   private

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grok-parser
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.6.2
 platform: ruby
 authors:
 - kiyoto
 - Kenji Okimoto
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-11-28 00:00:00.000000000 Z
+date: 2020-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -73,7 +73,7 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-description: 
+description:
 email:
 - kiyoto@treasure-data.com
 - okimoto@clear-code.com
@@ -123,7 +123,7 @@ homepage: https://github.com/fluent/fluent-plugin-grok-parser
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -138,9 +138,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Fluentd plugin to support Logstash-inspired Grok format for parsing logs
 test_files: