fluent-plugin-grok-parser 2.5.1 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +35 -3
- data/fluent-plugin-grok-parser.gemspec +1 -1
- data/lib/fluent/plugin/grok.rb +8 -0
- data/lib/fluent/plugin/parser_grok.rb +5 -0
- data/patterns/aws +3 -0
- data/patterns/firewalls +6 -1
- data/patterns/grok-patterns +5 -11
- data/patterns/java +2 -3
- data/patterns/redis +1 -1
- data/test/test_grok_parser.rb +40 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7bd9fc1deb8c5c630d6c8b446e203fdcab878e0f88c18f923795f54fad0993d9
|
4
|
+
data.tar.gz: fe990b8fcf889777bc1cb45f07262c495b3a3854ef3b126669a5aa543c1676d4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 27a9cdfde61baf4aa004cc5da4b8e6380f60ba9ebb39b21e01b2a9eeefd18718ef160afaf97c41710149bbf3997eb3ba82d25cfc88e42be69969319e4ab42257
|
7
|
+
data.tar.gz: aecafcd5fc7d35bc5c7137f98e5587f0f5931e04ac54b28435e931473423a92a08660628b034ec25009a55ece71cb034fa3f250501d94ac1e6e4ff18a77998e3
|
data/README.md
CHANGED
@@ -95,8 +95,7 @@ You can use this parser without `multiline_start_regexp` when you know your data
|
|
95
95
|
|
96
96
|
## Configurations
|
97
97
|
|
98
|
-
* See also: [
|
99
|
-
* See also: [Parser Plugin Overview](https://docs.fluentd.org/v1.0/articles/parser-plugin-overview)
|
98
|
+
* See also: [Config: Parse Section - Fluentd](https://docs.fluentd.org/configuration/parse-section)
|
100
99
|
|
101
100
|
* **time_format** (string) (optional): The format of the time field.
|
102
101
|
* **grok_pattern** (string) (optional): The pattern of grok. You cannot specify multiple grok pattern with this.
|
@@ -105,6 +104,17 @@ You can use this parser without `multiline_start_regexp` when you know your data
|
|
105
104
|
* **grok_name_key** (string) (optional): The key name to store grok section's name
|
106
105
|
* **multi_line_start_regexp** (string) (optional): The regexp to match beginning of multiline. This is only for "multiline_grok".
|
107
106
|
|
107
|
+
### \<grok\> section (optional) (multiple)
|
108
|
+
|
109
|
+
* **name** (string) (optional): The name of this grok section
|
110
|
+
* **pattern** (string) (required): The pattern of grok
|
111
|
+
* **keep_time_key** (bool) (optional): If true, keep time field in the record.
|
112
|
+
* **time_key** (string) (optional): Specify time field for event time. If the event doesn't have this field, current time is used.
|
113
|
+
* Default value: `time`.
|
114
|
+
* **time_format** (string) (optional): Process value using specified format. This is available only when time_type is string
|
115
|
+
* **timezone** (string) (optional): Use specified timezone. one can parse/format the time value in the specified timezone.
|
116
|
+
|
117
|
+
|
108
118
|
## Examples
|
109
119
|
|
110
120
|
### Using grok\_failure\_key
|
@@ -184,6 +194,28 @@ This will add keys like following:
|
|
184
194
|
Add `grokfailure` key to the record if the record does not match any grok pattern.
|
185
195
|
See also test code for more details.
|
186
196
|
|
197
|
+
## How to parse time value using specific timezone
|
198
|
+
|
199
|
+
```aconf
|
200
|
+
<source>
|
201
|
+
@type tail
|
202
|
+
path /path/to/log
|
203
|
+
tag grokked_log
|
204
|
+
<parse>
|
205
|
+
@type grok
|
206
|
+
<grok>
|
207
|
+
name mylog-without-timezone
|
208
|
+
pattern %{DATESTAMP:time} %{GREEDYDATE:message}
|
209
|
+
timezone Asia/Tokyo
|
210
|
+
</grok>
|
211
|
+
</parse>
|
212
|
+
</source>
|
213
|
+
```
|
214
|
+
|
215
|
+
This will parse the `time` value as "Asia/Tokyo" timezone.
|
216
|
+
|
217
|
+
See [Config: Parse Section - Fluentd](https://docs.fluentd.org/configuration/parse-section) for more details about timezone.
|
218
|
+
|
187
219
|
## How to write Grok patterns
|
188
220
|
|
189
221
|
Grok patterns look like `%{PATTERN_NAME:name}` where ":name" is optional. If "name" is provided, then it
|
@@ -271,7 +303,7 @@ Here is a sample config using the Grok parser with `in_tail` and the `types` par
|
|
271
303
|
|
272
304
|
If you want to use this plugin with Fluentd v0.12.x or earlier, you can use this plugin version v1.x.
|
273
305
|
|
274
|
-
See also: [Plugin Management | Fluentd](
|
306
|
+
See also: [Plugin Management | Fluentd](https://docs.fluentd.org/deployment/plugin-management)
|
275
307
|
|
276
308
|
## License
|
277
309
|
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-grok-parser"
|
7
|
-
spec.version = "2.
|
7
|
+
spec.version = "2.6.0"
|
8
8
|
spec.authors = ["kiyoto", "Kenji Okimoto"]
|
9
9
|
spec.email = ["kiyoto@treasure-data.com", "okimoto@clear-code.com"]
|
10
10
|
spec.summary = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}
|
data/lib/fluent/plugin/grok.rb
CHANGED
@@ -27,6 +27,8 @@ module Fluent
|
|
27
27
|
@multiline_mode = false
|
28
28
|
@conf = conf
|
29
29
|
@plugin = plugin
|
30
|
+
@time_format = nil
|
31
|
+
@timezone = nil
|
30
32
|
if @plugin.respond_to?(:firstline?)
|
31
33
|
@multiline_mode = true
|
32
34
|
end
|
@@ -39,6 +41,9 @@ module Fluent
|
|
39
41
|
if @plugin.respond_to?(:time_format)
|
40
42
|
@time_format = @plugin.time_format
|
41
43
|
end
|
44
|
+
if @plugin.respond_to?(:timezone)
|
45
|
+
@timezone = @plugin.timezone
|
46
|
+
end
|
42
47
|
end
|
43
48
|
|
44
49
|
def add_patterns_from_file(path)
|
@@ -105,6 +110,9 @@ module Fluent
|
|
105
110
|
if conf["time_format"] || @time_format
|
106
111
|
_conf["time_format"] = conf["time_format"] || @time_format
|
107
112
|
end
|
113
|
+
if conf["timezone"] || @timezone
|
114
|
+
_conf["timezone"] = conf["timezone"] || @timezone
|
115
|
+
end
|
108
116
|
_conf["expression"] = regexp
|
109
117
|
config = Fluent::Config::Element.new("parse", "", _conf, [])
|
110
118
|
parser = Fluent::Plugin::RegexpParser.new
|
@@ -22,9 +22,14 @@ module Fluent
|
|
22
22
|
config_param :name, :string, default: nil
|
23
23
|
desc "The pattern of grok"
|
24
24
|
config_param :pattern, :string
|
25
|
+
desc "If true, keep time field in the record."
|
25
26
|
config_param :keep_time_key, :bool, default: false
|
27
|
+
desc "Specify time field for event time. If the event doesn't have this field, current time is used."
|
26
28
|
config_param :time_key, :string, default: "time"
|
29
|
+
desc "Process value using specified format. This is available only when time_type is string"
|
27
30
|
config_param :time_format, :string, default: nil
|
31
|
+
desc "Use specified timezone. one can parse/format the time value in the specified timezone."
|
32
|
+
config_param :timezone, :string, default: nil
|
28
33
|
end
|
29
34
|
|
30
35
|
def initialize
|
data/patterns/aws
CHANGED
@@ -9,3 +9,6 @@ ELB_URI %{URIPROTO:proto}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{
|
|
9
9
|
ELB_REQUEST_LINE (?:%{WORD:verb} %{ELB_URI:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})
|
10
10
|
|
11
11
|
ELB_ACCESS_LOG %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:integer} (?:(%{IP:backendip}:?:%{INT:backendport:integer})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:integer} %{INT:backend_response:integer} %{INT:received_bytes:integer} %{INT:bytes:integer} "%{ELB_REQUEST_LINE}"
|
12
|
+
|
13
|
+
CLOUDFRONT_ACCESS_LOG (?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\t%{TIME})\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:integer}|-)\t%{IPORHOST:clientip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:cs_uri_stem}\t%{NUMBER:sc_status:integer}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:agent}\t%{GREEDYDATA:cs_uri_query}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:integer}\t%{GREEDYDATA:time_taken:float}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}
|
14
|
+
|
data/patterns/firewalls
CHANGED
@@ -36,7 +36,7 @@ CISCOFW106006_106007_106010 %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction}
|
|
36
36
|
# ASA-3-106014
|
37
37
|
CISCOFW106014 %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\(%{DATA:dst_fwuser}\))? \(type %{INT:icmp_type}, code %{INT:icmp_code}\)
|
38
38
|
# ASA-6-106015
|
39
|
-
CISCOFW106015 %{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags}
|
39
|
+
CISCOFW106015 %{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface}
|
40
40
|
# ASA-1-106021
|
41
41
|
CISCOFW106021 %{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface}
|
42
42
|
# ASA-4-106023
|
@@ -45,6 +45,8 @@ CISCOFW106023 %{CISCO_ACTION:action}( protocol)? %{WORD:protocol} src %{DATA:src
|
|
45
45
|
CISCOFW106100_2_3 access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} for user '%{DATA:src_fwuser}' %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\) -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\) hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]
|
46
46
|
# ASA-5-106100
|
47
47
|
CISCOFW106100 access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]
|
48
|
+
# ASA-5-304001
|
49
|
+
CISCOFW304001 %{IP:src_ip}(\(%{DATA:src_fwuser}\))? Accessed URL %{IP:dst_ip}:%{GREEDYDATA:dst_url}
|
48
50
|
# ASA-6-110002
|
49
51
|
CISCOFW110002 %{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port}
|
50
52
|
# ASA-6-302010
|
@@ -84,3 +86,6 @@ CISCOFW733100 \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Cu
|
|
84
86
|
# Shorewall firewall logs
|
85
87
|
SHOREWALL (%{SYSLOGTIMESTAMP:timestamp}) (%{WORD:nf_host}) kernel:.*Shorewall:(%{WORD:nf_action1})?:(%{WORD:nf_action2})?.*IN=(%{USERNAME:nf_in_interface})?.*(OUT= *MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?|OUT=%{USERNAME:nf_out_interface}).*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*LEN=(%{WORD:nf_len}).?*TOS=(%{WORD:nf_tos}).?*PREC=(%{WORD:nf_prec}).?*TTL=(%{INT:nf_ttl}).?*ID=(%{INT:nf_id}).?*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*)
|
86
88
|
#== End Shorewall
|
89
|
+
#== SuSE Firewall 2 ==
|
90
|
+
SFW2 ((%{SYSLOGTIMESTAMP})|(%{TIMESTAMP_ISO8601}))\s*%{HOSTNAME}\s*kernel\S+\s*%{NAGIOSTIME}\s*SFW2\-INext\-%{NOTSPACE:nf_action}\s*IN=%{USERNAME:nf_in_interface}.*OUT=((\s*%{USERNAME:nf_out_interface})|(\s*))MAC=((%{COMMONMAC:nf_dst_mac}:%{COMMONMAC:nf_src_mac})|(\s*)).*SRC=%{IP:nf_src_ip}\s*DST=%{IP:nf_dst_ip}.*PROTO=%{WORD:nf_protocol}((.*SPT=%{INT:nf_src_port}.*DPT=%{INT:nf_dst_port}.*)|())
|
91
|
+
#== End SuSE ==
|
data/patterns/grok-patterns
CHANGED
@@ -2,7 +2,6 @@ USERNAME [a-zA-Z0-9._-]+
|
|
2
2
|
USER %{USERNAME}
|
3
3
|
EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+
|
4
4
|
EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME}
|
5
|
-
HTTPDUSER %{EMAILADDRESS}|%{USER}
|
6
5
|
INT (?:[+-]?(?:[0-9]+))
|
7
6
|
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
|
8
7
|
NUMBER (?:%{BASE10NUM})
|
@@ -18,6 +17,8 @@ DATA .*?
|
|
18
17
|
GREEDYDATA .*
|
19
18
|
QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
|
20
19
|
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
|
20
|
+
# URN, allowing use of RFC 2141 section 2.3 reserved characters
|
21
|
+
URN urn:[0-9A-Za-z][0-9A-Za-z-]{0,31}:(?:%[0-9a-fA-F]{2}|[0-9A-Za-z()+,.:=@;$_!*'/?#-])+
|
21
22
|
|
22
23
|
# Networking
|
23
24
|
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
|
@@ -33,7 +34,7 @@ HOSTPORT %{IPORHOST}:%{POSINT}
|
|
33
34
|
|
34
35
|
# paths
|
35
36
|
PATH (?:%{UNIXPATH}|%{WINPATH})
|
36
|
-
UNIXPATH (/([\w_
|
37
|
+
UNIXPATH (/([\w_%!$@:.,+~-]+|\\.)*)+
|
37
38
|
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
|
38
39
|
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
|
39
40
|
URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
|
@@ -47,7 +48,7 @@ URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
|
|
47
48
|
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
|
48
49
|
|
49
50
|
# Months: January, Feb, 3, 03, 12, December
|
50
|
-
MONTH \b(?:
|
51
|
+
MONTH \b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\b
|
51
52
|
MONTHNUM (?:0?[1-9]|1[0-2])
|
52
53
|
MONTHNUM2 (?:0[1-9]|1[0-2])
|
53
54
|
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
|
@@ -70,12 +71,11 @@ ISO8601_SECOND (?:%{SECOND}|60)
|
|
70
71
|
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
|
71
72
|
DATE %{DATE_US}|%{DATE_EU}
|
72
73
|
DATESTAMP %{DATE}[- ]%{TIME}
|
73
|
-
TZ (?:[
|
74
|
+
TZ (?:[APMCE][SD]T|UTC)
|
74
75
|
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
|
75
76
|
DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
|
76
77
|
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
|
77
78
|
DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
|
78
|
-
HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
|
79
79
|
|
80
80
|
# Syslog Dates: Month Day HH:MM:SS
|
81
81
|
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
|
@@ -90,12 +90,6 @@ QS %{QUOTEDSTRING}
|
|
90
90
|
|
91
91
|
# Log formats
|
92
92
|
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
|
93
|
-
COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
|
94
|
-
COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
|
95
|
-
HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}
|
96
|
-
HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
|
97
|
-
HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
|
98
|
-
|
99
93
|
|
100
94
|
# Log Levels
|
101
95
|
LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
|
data/patterns/java
CHANGED
@@ -1,15 +1,14 @@
|
|
1
1
|
JAVACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]*
|
2
2
|
#Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source'
|
3
3
|
JAVAFILE (?:[A-Za-z0-9_. -]+)
|
4
|
-
#Allow special <init>
|
5
|
-
JAVAMETHOD (?:(<init>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
|
4
|
+
#Allow special <init>, <clinit> methods
|
5
|
+
JAVAMETHOD (?:(<(?:cl)?init>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
|
6
6
|
#Line number is optional in special cases 'Native method' or 'Unknown source'
|
7
7
|
JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\)
|
8
8
|
# Java Logs
|
9
9
|
JAVATHREAD (?:[A-Z]{2}-Processor[\d]+)
|
10
10
|
JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+
|
11
11
|
JAVAFILE (?:[A-Za-z0-9_.-]+)
|
12
|
-
JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\)
|
13
12
|
JAVALOGMESSAGE (.*)
|
14
13
|
# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM
|
15
14
|
CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)
|
data/patterns/redis
CHANGED
data/test/test_grok_parser.rb
CHANGED
@@ -374,6 +374,46 @@ class GrokParserTest < ::Test::Unit::TestCase
|
|
374
374
|
assert_equal(expected_record, record)
|
375
375
|
end
|
376
376
|
end
|
377
|
+
|
378
|
+
test "timezone" do
|
379
|
+
d = create_driver(%[
|
380
|
+
<grok>
|
381
|
+
pattern %{TIMESTAMP_ISO8601:time} %{GREEDYDATA:message}
|
382
|
+
time_key time
|
383
|
+
time_format %Y-%m-%d %H:%M:%S
|
384
|
+
timezone Europe/Berlin
|
385
|
+
</grok>
|
386
|
+
])
|
387
|
+
d.instance.parse("2019-02-01 12:34:56 This is test") do |time, record|
|
388
|
+
assert_equal(event_time("2019-02-01 12:34:56 +0100"), time)
|
389
|
+
assert_equal({ "message" => "This is test" }, record)
|
390
|
+
end
|
391
|
+
end
|
392
|
+
|
393
|
+
test "multiple timezone" do
|
394
|
+
d = create_driver(%[
|
395
|
+
<grok>
|
396
|
+
pattern %{TIMESTAMP_ISO8601:time} 1 %{GREEDYDATA:message}
|
397
|
+
time_key time
|
398
|
+
time_format %Y-%m-%d %H:%M:%S
|
399
|
+
timezone Europe/Berlin
|
400
|
+
</grok>
|
401
|
+
<grok>
|
402
|
+
pattern %{TIMESTAMP_ISO8601:time} 2 %{GREEDYDATA:message}
|
403
|
+
time_key time
|
404
|
+
time_format %Y-%m-%d %H:%M:%S
|
405
|
+
timezone Asia/Aden
|
406
|
+
</grok>
|
407
|
+
])
|
408
|
+
d.instance.parse("2019-02-01 12:34:56 1 This is test") do |time, record|
|
409
|
+
assert_equal(event_time("2019-02-01 12:34:56 +0100"), time)
|
410
|
+
assert_equal({ "message" => "This is test" }, record)
|
411
|
+
end
|
412
|
+
d.instance.parse("2019-02-01 12:34:56 2 This is test") do |time, record|
|
413
|
+
assert_equal(event_time("2019-02-01 12:34:56 +0300"), time)
|
414
|
+
assert_equal({ "message" => "This is test" }, record)
|
415
|
+
end
|
416
|
+
end
|
377
417
|
end
|
378
418
|
|
379
419
|
private
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-grok-parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- kiyoto
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-
|
12
|
+
date: 2019-07-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|