fluent-plugin-grok-parser 2.5.1 → 2.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +35 -3
- data/fluent-plugin-grok-parser.gemspec +1 -1
- data/lib/fluent/plugin/grok.rb +8 -0
- data/lib/fluent/plugin/parser_grok.rb +5 -0
- data/patterns/aws +3 -0
- data/patterns/firewalls +6 -1
- data/patterns/grok-patterns +5 -11
- data/patterns/java +2 -3
- data/patterns/redis +1 -1
- data/test/test_grok_parser.rb +40 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7bd9fc1deb8c5c630d6c8b446e203fdcab878e0f88c18f923795f54fad0993d9
|
4
|
+
data.tar.gz: fe990b8fcf889777bc1cb45f07262c495b3a3854ef3b126669a5aa543c1676d4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 27a9cdfde61baf4aa004cc5da4b8e6380f60ba9ebb39b21e01b2a9eeefd18718ef160afaf97c41710149bbf3997eb3ba82d25cfc88e42be69969319e4ab42257
|
7
|
+
data.tar.gz: aecafcd5fc7d35bc5c7137f98e5587f0f5931e04ac54b28435e931473423a92a08660628b034ec25009a55ece71cb034fa3f250501d94ac1e6e4ff18a77998e3
|
data/README.md
CHANGED
@@ -95,8 +95,7 @@ You can use this parser without `multiline_start_regexp` when you know your data
|
|
95
95
|
|
96
96
|
## Configurations
|
97
97
|
|
98
|
-
* See also: [
|
99
|
-
* See also: [Parser Plugin Overview](https://docs.fluentd.org/v1.0/articles/parser-plugin-overview)
|
98
|
+
* See also: [Config: Parse Section - Fluentd](https://docs.fluentd.org/configuration/parse-section)
|
100
99
|
|
101
100
|
* **time_format** (string) (optional): The format of the time field.
|
102
101
|
* **grok_pattern** (string) (optional): The pattern of grok. You cannot specify multiple grok pattern with this.
|
@@ -105,6 +104,17 @@ You can use this parser without `multiline_start_regexp` when you know your data
|
|
105
104
|
* **grok_name_key** (string) (optional): The key name to store grok section's name
|
106
105
|
* **multi_line_start_regexp** (string) (optional): The regexp to match beginning of multiline. This is only for "multiline_grok".
|
107
106
|
|
107
|
+
### \<grok\> section (optional) (multiple)
|
108
|
+
|
109
|
+
* **name** (string) (optional): The name of this grok section
|
110
|
+
* **pattern** (string) (required): The pattern of grok
|
111
|
+
* **keep_time_key** (bool) (optional): If true, keep time field in the record.
|
112
|
+
* **time_key** (string) (optional): Specify time field for event time. If the event doesn't have this field, current time is used.
|
113
|
+
* Default value: `time`.
|
114
|
+
* **time_format** (string) (optional): Process value using specified format. This is available only when time_type is string
|
115
|
+
* **timezone** (string) (optional): Use specified timezone. one can parse/format the time value in the specified timezone.
|
116
|
+
|
117
|
+
|
108
118
|
## Examples
|
109
119
|
|
110
120
|
### Using grok\_failure\_key
|
@@ -184,6 +194,28 @@ This will add keys like following:
|
|
184
194
|
Add `grokfailure` key to the record if the record does not match any grok pattern.
|
185
195
|
See also test code for more details.
|
186
196
|
|
197
|
+
## How to parse time value using specific timezone
|
198
|
+
|
199
|
+
```aconf
|
200
|
+
<source>
|
201
|
+
@type tail
|
202
|
+
path /path/to/log
|
203
|
+
tag grokked_log
|
204
|
+
<parse>
|
205
|
+
@type grok
|
206
|
+
<grok>
|
207
|
+
name mylog-without-timezone
|
208
|
+
pattern %{DATESTAMP:time} %{GREEDYDATE:message}
|
209
|
+
timezone Asia/Tokyo
|
210
|
+
</grok>
|
211
|
+
</parse>
|
212
|
+
</source>
|
213
|
+
```
|
214
|
+
|
215
|
+
This will parse the `time` value as "Asia/Tokyo" timezone.
|
216
|
+
|
217
|
+
See [Config: Parse Section - Fluentd](https://docs.fluentd.org/configuration/parse-section) for more details about timezone.
|
218
|
+
|
187
219
|
## How to write Grok patterns
|
188
220
|
|
189
221
|
Grok patterns look like `%{PATTERN_NAME:name}` where ":name" is optional. If "name" is provided, then it
|
@@ -271,7 +303,7 @@ Here is a sample config using the Grok parser with `in_tail` and the `types` par
|
|
271
303
|
|
272
304
|
If you want to use this plugin with Fluentd v0.12.x or earlier, you can use this plugin version v1.x.
|
273
305
|
|
274
|
-
See also: [Plugin Management | Fluentd](
|
306
|
+
See also: [Plugin Management | Fluentd](https://docs.fluentd.org/deployment/plugin-management)
|
275
307
|
|
276
308
|
## License
|
277
309
|
|
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-grok-parser"
|
7
|
-
spec.version = "2.
|
7
|
+
spec.version = "2.6.0"
|
8
8
|
spec.authors = ["kiyoto", "Kenji Okimoto"]
|
9
9
|
spec.email = ["kiyoto@treasure-data.com", "okimoto@clear-code.com"]
|
10
10
|
spec.summary = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}
|
data/lib/fluent/plugin/grok.rb
CHANGED
@@ -27,6 +27,8 @@ module Fluent
|
|
27
27
|
@multiline_mode = false
|
28
28
|
@conf = conf
|
29
29
|
@plugin = plugin
|
30
|
+
@time_format = nil
|
31
|
+
@timezone = nil
|
30
32
|
if @plugin.respond_to?(:firstline?)
|
31
33
|
@multiline_mode = true
|
32
34
|
end
|
@@ -39,6 +41,9 @@ module Fluent
|
|
39
41
|
if @plugin.respond_to?(:time_format)
|
40
42
|
@time_format = @plugin.time_format
|
41
43
|
end
|
44
|
+
if @plugin.respond_to?(:timezone)
|
45
|
+
@timezone = @plugin.timezone
|
46
|
+
end
|
42
47
|
end
|
43
48
|
|
44
49
|
def add_patterns_from_file(path)
|
@@ -105,6 +110,9 @@ module Fluent
|
|
105
110
|
if conf["time_format"] || @time_format
|
106
111
|
_conf["time_format"] = conf["time_format"] || @time_format
|
107
112
|
end
|
113
|
+
if conf["timezone"] || @timezone
|
114
|
+
_conf["timezone"] = conf["timezone"] || @timezone
|
115
|
+
end
|
108
116
|
_conf["expression"] = regexp
|
109
117
|
config = Fluent::Config::Element.new("parse", "", _conf, [])
|
110
118
|
parser = Fluent::Plugin::RegexpParser.new
|
@@ -22,9 +22,14 @@ module Fluent
|
|
22
22
|
config_param :name, :string, default: nil
|
23
23
|
desc "The pattern of grok"
|
24
24
|
config_param :pattern, :string
|
25
|
+
desc "If true, keep time field in the record."
|
25
26
|
config_param :keep_time_key, :bool, default: false
|
27
|
+
desc "Specify time field for event time. If the event doesn't have this field, current time is used."
|
26
28
|
config_param :time_key, :string, default: "time"
|
29
|
+
desc "Process value using specified format. This is available only when time_type is string"
|
27
30
|
config_param :time_format, :string, default: nil
|
31
|
+
desc "Use specified timezone. one can parse/format the time value in the specified timezone."
|
32
|
+
config_param :timezone, :string, default: nil
|
28
33
|
end
|
29
34
|
|
30
35
|
def initialize
|
data/patterns/aws
CHANGED
@@ -9,3 +9,6 @@ ELB_URI %{URIPROTO:proto}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST:urihost})?(?:%{
|
|
9
9
|
ELB_REQUEST_LINE (?:%{WORD:verb} %{ELB_URI:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})
|
10
10
|
|
11
11
|
ELB_ACCESS_LOG %{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:elb} %{IP:clientip}:%{INT:clientport:integer} (?:(%{IP:backendip}:?:%{INT:backendport:integer})|-) %{NUMBER:request_processing_time:float} %{NUMBER:backend_processing_time:float} %{NUMBER:response_processing_time:float} %{INT:response:integer} %{INT:backend_response:integer} %{INT:received_bytes:integer} %{INT:bytes:integer} "%{ELB_REQUEST_LINE}"
|
12
|
+
|
13
|
+
CLOUDFRONT_ACCESS_LOG (?<timestamp>%{YEAR}-%{MONTHNUM}-%{MONTHDAY}\t%{TIME})\t%{WORD:x_edge_location}\t(?:%{NUMBER:sc_bytes:integer}|-)\t%{IPORHOST:clientip}\t%{WORD:cs_method}\t%{HOSTNAME:cs_host}\t%{NOTSPACE:cs_uri_stem}\t%{NUMBER:sc_status:integer}\t%{GREEDYDATA:referrer}\t%{GREEDYDATA:agent}\t%{GREEDYDATA:cs_uri_query}\t%{GREEDYDATA:cookies}\t%{WORD:x_edge_result_type}\t%{NOTSPACE:x_edge_request_id}\t%{HOSTNAME:x_host_header}\t%{URIPROTO:cs_protocol}\t%{INT:cs_bytes:integer}\t%{GREEDYDATA:time_taken:float}\t%{GREEDYDATA:x_forwarded_for}\t%{GREEDYDATA:ssl_protocol}\t%{GREEDYDATA:ssl_cipher}\t%{GREEDYDATA:x_edge_response_result_type}
|
14
|
+
|
data/patterns/firewalls
CHANGED
@@ -36,7 +36,7 @@ CISCOFW106006_106007_106010 %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction}
|
|
36
36
|
# ASA-3-106014
|
37
37
|
CISCOFW106014 %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IP:dst_ip}(\(%{DATA:dst_fwuser}\))? \(type %{INT:icmp_type}, code %{INT:icmp_code}\)
|
38
38
|
# ASA-6-106015
|
39
|
-
CISCOFW106015 %{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags}
|
39
|
+
CISCOFW106015 %{CISCO_ACTION:action} %{WORD:protocol} \(%{DATA:policy_id}\) from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface}
|
40
40
|
# ASA-1-106021
|
41
41
|
CISCOFW106021 %{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface}
|
42
42
|
# ASA-4-106023
|
@@ -45,6 +45,8 @@ CISCOFW106023 %{CISCO_ACTION:action}( protocol)? %{WORD:protocol} src %{DATA:src
|
|
45
45
|
CISCOFW106100_2_3 access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} for user '%{DATA:src_fwuser}' %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\) -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\) hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]
|
46
46
|
# ASA-5-106100
|
47
47
|
CISCOFW106100 access-list %{NOTSPACE:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}\(%{INT:src_port}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IP:dst_ip}\(%{INT:dst_port}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]
|
48
|
+
# ASA-5-304001
|
49
|
+
CISCOFW304001 %{IP:src_ip}(\(%{DATA:src_fwuser}\))? Accessed URL %{IP:dst_ip}:%{GREEDYDATA:dst_url}
|
48
50
|
# ASA-6-110002
|
49
51
|
CISCOFW110002 %{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port}
|
50
52
|
# ASA-6-302010
|
@@ -84,3 +86,6 @@ CISCOFW733100 \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Cu
|
|
84
86
|
# Shorewall firewall logs
|
85
87
|
SHOREWALL (%{SYSLOGTIMESTAMP:timestamp}) (%{WORD:nf_host}) kernel:.*Shorewall:(%{WORD:nf_action1})?:(%{WORD:nf_action2})?.*IN=(%{USERNAME:nf_in_interface})?.*(OUT= *MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?|OUT=%{USERNAME:nf_out_interface}).*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*LEN=(%{WORD:nf_len}).?*TOS=(%{WORD:nf_tos}).?*PREC=(%{WORD:nf_prec}).?*TTL=(%{INT:nf_ttl}).?*ID=(%{INT:nf_id}).?*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*)
|
86
88
|
#== End Shorewall
|
89
|
+
#== SuSE Firewall 2 ==
|
90
|
+
SFW2 ((%{SYSLOGTIMESTAMP})|(%{TIMESTAMP_ISO8601}))\s*%{HOSTNAME}\s*kernel\S+\s*%{NAGIOSTIME}\s*SFW2\-INext\-%{NOTSPACE:nf_action}\s*IN=%{USERNAME:nf_in_interface}.*OUT=((\s*%{USERNAME:nf_out_interface})|(\s*))MAC=((%{COMMONMAC:nf_dst_mac}:%{COMMONMAC:nf_src_mac})|(\s*)).*SRC=%{IP:nf_src_ip}\s*DST=%{IP:nf_dst_ip}.*PROTO=%{WORD:nf_protocol}((.*SPT=%{INT:nf_src_port}.*DPT=%{INT:nf_dst_port}.*)|())
|
91
|
+
#== End SuSE ==
|
data/patterns/grok-patterns
CHANGED
@@ -2,7 +2,6 @@ USERNAME [a-zA-Z0-9._-]+
|
|
2
2
|
USER %{USERNAME}
|
3
3
|
EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+
|
4
4
|
EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME}
|
5
|
-
HTTPDUSER %{EMAILADDRESS}|%{USER}
|
6
5
|
INT (?:[+-]?(?:[0-9]+))
|
7
6
|
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
|
8
7
|
NUMBER (?:%{BASE10NUM})
|
@@ -18,6 +17,8 @@ DATA .*?
|
|
18
17
|
GREEDYDATA .*
|
19
18
|
QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
|
20
19
|
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
|
20
|
+
# URN, allowing use of RFC 2141 section 2.3 reserved characters
|
21
|
+
URN urn:[0-9A-Za-z][0-9A-Za-z-]{0,31}:(?:%[0-9a-fA-F]{2}|[0-9A-Za-z()+,.:=@;$_!*'/?#-])+
|
21
22
|
|
22
23
|
# Networking
|
23
24
|
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
|
@@ -33,7 +34,7 @@ HOSTPORT %{IPORHOST}:%{POSINT}
|
|
33
34
|
|
34
35
|
# paths
|
35
36
|
PATH (?:%{UNIXPATH}|%{WINPATH})
|
36
|
-
UNIXPATH (/([\w_
|
37
|
+
UNIXPATH (/([\w_%!$@:.,+~-]+|\\.)*)+
|
37
38
|
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
|
38
39
|
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
|
39
40
|
URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
|
@@ -47,7 +48,7 @@ URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
|
|
47
48
|
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
|
48
49
|
|
49
50
|
# Months: January, Feb, 3, 03, 12, December
|
50
|
-
MONTH \b(?:
|
51
|
+
MONTH \b(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)\b
|
51
52
|
MONTHNUM (?:0?[1-9]|1[0-2])
|
52
53
|
MONTHNUM2 (?:0[1-9]|1[0-2])
|
53
54
|
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
|
@@ -70,12 +71,11 @@ ISO8601_SECOND (?:%{SECOND}|60)
|
|
70
71
|
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
|
71
72
|
DATE %{DATE_US}|%{DATE_EU}
|
72
73
|
DATESTAMP %{DATE}[- ]%{TIME}
|
73
|
-
TZ (?:[
|
74
|
+
TZ (?:[APMCE][SD]T|UTC)
|
74
75
|
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
|
75
76
|
DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
|
76
77
|
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
|
77
78
|
DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
|
78
|
-
HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
|
79
79
|
|
80
80
|
# Syslog Dates: Month Day HH:MM:SS
|
81
81
|
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
|
@@ -90,12 +90,6 @@ QS %{QUOTEDSTRING}
|
|
90
90
|
|
91
91
|
# Log formats
|
92
92
|
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
|
93
|
-
COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
|
94
|
-
COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
|
95
|
-
HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}
|
96
|
-
HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
|
97
|
-
HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
|
98
|
-
|
99
93
|
|
100
94
|
# Log Levels
|
101
95
|
LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
|
data/patterns/java
CHANGED
@@ -1,15 +1,14 @@
|
|
1
1
|
JAVACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]*
|
2
2
|
#Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source'
|
3
3
|
JAVAFILE (?:[A-Za-z0-9_. -]+)
|
4
|
-
#Allow special <init>
|
5
|
-
JAVAMETHOD (?:(<init>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
|
4
|
+
#Allow special <init>, <clinit> methods
|
5
|
+
JAVAMETHOD (?:(<(?:cl)?init>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
|
6
6
|
#Line number is optional in special cases 'Native method' or 'Unknown source'
|
7
7
|
JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\)
|
8
8
|
# Java Logs
|
9
9
|
JAVATHREAD (?:[A-Z]{2}-Processor[\d]+)
|
10
10
|
JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+
|
11
11
|
JAVAFILE (?:[A-Za-z0-9_.-]+)
|
12
|
-
JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\)
|
13
12
|
JAVALOGMESSAGE (.*)
|
14
13
|
# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM
|
15
14
|
CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)
|
data/patterns/redis
CHANGED
data/test/test_grok_parser.rb
CHANGED
@@ -374,6 +374,46 @@ class GrokParserTest < ::Test::Unit::TestCase
|
|
374
374
|
assert_equal(expected_record, record)
|
375
375
|
end
|
376
376
|
end
|
377
|
+
|
378
|
+
test "timezone" do
|
379
|
+
d = create_driver(%[
|
380
|
+
<grok>
|
381
|
+
pattern %{TIMESTAMP_ISO8601:time} %{GREEDYDATA:message}
|
382
|
+
time_key time
|
383
|
+
time_format %Y-%m-%d %H:%M:%S
|
384
|
+
timezone Europe/Berlin
|
385
|
+
</grok>
|
386
|
+
])
|
387
|
+
d.instance.parse("2019-02-01 12:34:56 This is test") do |time, record|
|
388
|
+
assert_equal(event_time("2019-02-01 12:34:56 +0100"), time)
|
389
|
+
assert_equal({ "message" => "This is test" }, record)
|
390
|
+
end
|
391
|
+
end
|
392
|
+
|
393
|
+
test "multiple timezone" do
|
394
|
+
d = create_driver(%[
|
395
|
+
<grok>
|
396
|
+
pattern %{TIMESTAMP_ISO8601:time} 1 %{GREEDYDATA:message}
|
397
|
+
time_key time
|
398
|
+
time_format %Y-%m-%d %H:%M:%S
|
399
|
+
timezone Europe/Berlin
|
400
|
+
</grok>
|
401
|
+
<grok>
|
402
|
+
pattern %{TIMESTAMP_ISO8601:time} 2 %{GREEDYDATA:message}
|
403
|
+
time_key time
|
404
|
+
time_format %Y-%m-%d %H:%M:%S
|
405
|
+
timezone Asia/Aden
|
406
|
+
</grok>
|
407
|
+
])
|
408
|
+
d.instance.parse("2019-02-01 12:34:56 1 This is test") do |time, record|
|
409
|
+
assert_equal(event_time("2019-02-01 12:34:56 +0100"), time)
|
410
|
+
assert_equal({ "message" => "This is test" }, record)
|
411
|
+
end
|
412
|
+
d.instance.parse("2019-02-01 12:34:56 2 This is test") do |time, record|
|
413
|
+
assert_equal(event_time("2019-02-01 12:34:56 +0300"), time)
|
414
|
+
assert_equal({ "message" => "This is test" }, record)
|
415
|
+
end
|
416
|
+
end
|
377
417
|
end
|
378
418
|
|
379
419
|
private
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-grok-parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- kiyoto
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-
|
12
|
+
date: 2019-07-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|