logstash-patterns-core 4.0.1 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -56,4 +56,21 @@ describe "HAPROXY" do
56
56
 
57
57
  end
58
58
 
59
+ context "Parsing HAPROXY log line that is truncated and thus not ending with a double quote or HTTP version." do
60
+
61
+ let(:value) { 'Jul 31 22:20:22 loadbalancer haproxy[1190]: 203.0.113.54:59968 [31/Jul/2017:22:20:22.447] loadbalancer default/instance8 135/0/1/19/156 200 1015 - - --VR 8/8/0/0/0 0/0 "GET /path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss'}
62
+ subject { grok_match(haproxyhttpbase_pattern, value)}
63
+
64
+ it { should include("client_ip" => "203.0.113.54") }
65
+ it { should include("http_verb" => "GET") }
66
+ it { should include("server_name" => "instance8") }
67
+ it { should include("http_request" => "/path/to/request/that/exceeds/more/than/1024/characterssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssssss") }
68
+ it { should_not have_key("http_version") }
69
+
70
+ it "generates a message field" do
71
+ expect(subject["message"]).to include("loadbalancer default/instance8")
72
+ end
73
+
74
+ end
75
+
59
76
  end
@@ -2,23 +2,183 @@
2
2
  require "spec_helper"
3
3
  require "logstash/patterns/core"
4
4
 
5
+ describe "HTTPD_COMBINEDLOG" do
6
+
7
+ let(:pattern) { 'HTTPD_COMBINEDLOG' }
8
+ let(:grok) { grok_match(pattern, message) }
9
+
10
+ context "typical test case" do
11
+
12
+ let(:message) { '83.149.9.216 - - [24/Feb/2015:23:13:42 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'}
13
+
14
+ it "matches" do
15
+ expect(grok).to include(
16
+ 'clientip' => '83.149.9.216',
17
+ 'verb' => 'GET',
18
+ 'request' => '/presentations/logstash-monitorama-2013/images/kibana-search.png',
19
+ 'httpversion' => '1.1',
20
+ 'response' => '200',
21
+ 'bytes' => '203023',
22
+ 'referrer' => '"http://semicomplete.com/presentations/logstash-monitorama-2013/"',
23
+ 'agent' => '"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"'
24
+ )
25
+ end
26
+
27
+ it "does not capture 'null' fields" do
28
+ expect(grok).to include('auth' => '-', 'ident' => '-')
29
+ end
30
+
31
+ end
32
+
33
+ context "email address in auth field" do
34
+
35
+ let(:message) { '10.0.0.1 - username@example.com [07/Apr/2016:18:42:24 +0000] "GET /bar/foo/users/1/username%40example.com/authenticate?token=blargh&client_id=15 HTTP/1.1" 400 75 "" "Mozilla/5.0 (iPad; CPU OS 9_3_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13E238 Safari/601.1"'}
36
+
37
+ it "gets captured" do
38
+ expect(grok).to include("auth" => "username@example.com")
39
+ end
40
+
41
+ end
42
+
43
+ context 'sample OPTIONS line' do
44
+
45
+ let(:message) { '83.149.9.216 - a.user [11/Jan/2020:23:05:27 +0100] "OPTIONS /remote.php/ HTTP/1.1" - 7908 "-" "monitoring-client (v2.2)"' }
46
+
47
+ it 'matches' do
48
+ expect(grok).to include("verb" => "OPTIONS", 'request' => '/remote.php/', 'httpversion' => '1.1', "bytes" => '7908')
49
+ end
50
+
51
+ it 'does not capture optional response code' do
52
+ expect(grok.keys).to_not include("response")
53
+ end
54
+
55
+ end
56
+
57
+ end
58
+
5
59
  describe "HTTPD_ERRORLOG" do
6
60
 
7
- it "matches a full httpd 2.4 message" do
8
- expect(subject).to match("[Mon Aug 31 09:30:48.958285 2015] [proxy_fcgi:error] [pid 28787:tid 140169587934976] (70008)Partial results are valid but processing is incomplete: [client 58.13.45.166:59307] AH01075: Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product")
61
+ let(:pattern) { 'HTTPD_ERRORLOG' }
62
+ let(:grok) { grok_match(pattern, message) }
63
+
64
+ context "matches a full httpd 2.4 message" do
65
+ let(:message) do
66
+ "[Mon Aug 31 09:30:48.958285 2015] [proxy_fcgi:error] [pid 28787:tid 140169587934976] (70008)Partial results are valid but processing is incomplete: [client 58.13.45.166:59307] AH01075: Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product"
67
+ end
68
+ it "generates the fields" do
69
+
70
+ expect(grok).to include(
71
+ 'timestamp' => 'Mon Aug 31 09:30:48.958285 2015',
72
+ 'module' => 'proxy_fcgi',
73
+ 'loglevel' => 'error',
74
+ 'pid' => '28787',
75
+ 'tid' => '140169587934976',
76
+ 'proxy_errorcode' => '70008',
77
+ 'proxy_message' => 'Partial results are valid but processing is incomplete',
78
+ 'clientip' => '58.13.45.166',
79
+ 'clientport' => '59307',
80
+ 'errorcode' => 'AH01075',
81
+ 'message' => [ message, 'Error dispatching request to : (reading input brigade), referer: http://example.com/index.php?id_product=11&controller=product' ],
82
+ )
83
+ end
9
84
  end
10
85
 
11
- it "matches a httpd 2.2 log message" do
12
- expect(subject).to match("[Mon Aug 31 16:27:04 2015] [error] [client 10.17.42.3] Premature end of script headers: example.com")
86
+ context "HTTPD_ERRORLOG", "matches a httpd 2.2 log message" do
87
+ let(:message) do
88
+ "[Mon Aug 31 16:27:04 2015] [error] [client 10.17.42.3] Premature end of script headers: example.com"
89
+ end
90
+ it "generates the fields" do
91
+ expect(grok).to include(
92
+ 'timestamp' => 'Mon Aug 31 16:27:04 2015',
93
+ 'loglevel' => 'error',
94
+ 'clientip' => '10.17.42.3',
95
+ 'message' => [ message, 'Premature end of script headers: example.com' ]
96
+ )
97
+ end
13
98
  end
14
99
 
15
- it "matches a short httpd 2.4 message" do
16
- expect(subject).to match("[Mon Aug 31 07:15:38.664897 2015] [proxy_fcgi:error] [pid 28786:tid 140169629898496] [client 81.139.1.34:52042] AH01071: Got error 'Primary script unknown\n'")
100
+ context "HTTPD_ERRORLOG", "a short httpd 2.4 message" do
101
+ let(:value1) {
102
+ "[Mon Aug 31 07:15:38.664897 2015] [proxy_fcgi:error] [pid 28786:tid 140169629898496] [client 81.139.1.34:52042] AH01071: Got error 'Primary script unknown\n'"
103
+ }
104
+ it "generates the fields" do
105
+ expect(grok_match(subject, value1)).to include(
106
+ 'timestamp' => 'Mon Aug 31 07:15:38.664897 2015',
107
+ 'module' => 'proxy_fcgi',
108
+ 'loglevel' => 'error',
109
+ 'pid' => '28786',
110
+ 'tid' => '140169629898496',
111
+ 'clientip' => '81.139.1.34',
112
+ 'clientport' => '52042',
113
+ 'errorcode' => 'AH01071',
114
+ 'message' => [ value1, "Got error 'Primary script unknown\n'" ]
115
+ )
116
+ end
117
+
118
+ let(:value2) {
119
+ "[Thu Apr 27 10:39:46.719636 2017] [php7:notice] [pid 17] [client 10.255.0.3:49580] Test error log record"
120
+ }
121
+ it "generates the fields" do
122
+ expect(grok_match(subject, value2)).to include(
123
+ 'timestamp' => 'Thu Apr 27 10:39:46.719636 2017',
124
+ 'module' => 'php7',
125
+ 'loglevel' => 'notice',
126
+ 'pid' => '17',
127
+ 'clientip' => '10.255.0.3',
128
+ 'clientport' => '49580',
129
+ 'message' => [ value2, "Test error log record" ]
130
+ )
131
+ end
17
132
  end
18
133
 
19
- it "matches an httpd 2.4 restart" do
20
- expect(subject).to match("[Mon Aug 31 06:29:47.406518 2015] [mpm_event:notice] [pid 24968:tid 140169861986176] AH00489: Apache/2.4.16 (Ubuntu) configured -- resuming normal operations")
21
- expect(subject).to match("[Mon Aug 31 06:29:47.406530 2015] [core:notice] [pid 24968:tid 140169861986176] AH00094: Command line: '/usr/sbin/apache2'")
134
+ context "HTTPD_ERRORLOG", "a httpd 2.4 restart message" do
135
+ let(:value1) {
136
+ "[Mon Aug 31 06:29:47.406518 2015] [mpm_event:notice] [pid 24968:tid 140169861986176] AH00489: Apache/2.4.16 (Ubuntu) configured -- resuming normal operations"
137
+ }
138
+ it "generates the fields" do
139
+ expect(grok_match(subject, value1)).to include(
140
+ 'timestamp' => 'Mon Aug 31 06:29:47.406518 2015',
141
+ 'module' => 'mpm_event',
142
+ 'loglevel' => 'notice',
143
+ 'pid' => '24968',
144
+ 'tid' => '140169861986176',
145
+ 'errorcode' => 'AH00489',
146
+ 'message' => [ value1, 'Apache/2.4.16 (Ubuntu) configured -- resuming normal operations' ]
147
+ )
148
+ end
149
+
150
+ let(:value2) {
151
+ "[Mon Aug 31 06:29:47.406530 2015] [core:notice] [pid 24968:tid 140169861986176] AH00094: Command line: '/usr/sbin/apache2'"
152
+ }
153
+ it "generates the fields" do
154
+ expect(grok_match(subject, value2)).to include(
155
+ 'timestamp' => 'Mon Aug 31 06:29:47.406530 2015',
156
+ 'module' => 'core',
157
+ 'loglevel' => 'notice',
158
+ 'pid' => '24968',
159
+ 'tid' => '140169861986176',
160
+ 'errorcode' => 'AH00094',
161
+ 'message' => [ value2, 'Command line: \'/usr/sbin/apache2\'' ]
162
+ )
163
+ end
22
164
  end
23
165
 
166
+ context 'a debug message' do
167
+ let(:message) do
168
+ '[Fri Feb 01 22:03:08.319124 2019] [authz_core:debug] [pid 9:tid 140597881775872] mod_authz_core.c(820): [client 172.17.0.1:50752] AH01626: authorization result of <RequireAny>: granted'
169
+ end
170
+
171
+ it 'matches imperfectly (legacy)' do
172
+ expect(grok).to include({
173
+ "timestamp"=>"Fri Feb 01 22:03:08.319124 2019",
174
+ "module"=>"authz_core",
175
+ "loglevel"=>"debug",
176
+ "pid"=>"9",
177
+ "tid"=>"140597881775872",
178
+ "errorcode"=>"mod_authz_core.c(820)",
179
+ "message"=>[message, "[client 172.17.0.1:50752] AH01626: authorization result of <RequireAny>: granted"]
180
+ })
181
+ end
182
+ end
183
+
24
184
  end
@@ -0,0 +1,45 @@
1
+ # encoding: utf-8
2
+ require "spec_helper"
3
+ require "logstash/patterns/core"
4
+
5
+ describe "JAVA" do
6
+ describe "JAVACLASS" do
7
+ let(:example) { 'hudson.node_monitors.AbstractAsyncNodeMonitorDescriptor' }
8
+ it "matches a java class with underscores" do
9
+ expect(grok_match(subject, example, true)['tags']).to be_nil
10
+ end
11
+ end
12
+ describe "JAVAFILE" do
13
+ let(:example) { 'Native Method' }
14
+ it "matches a java file name with spaces" do
15
+ expect(grok_match(subject, example, true)['tags']).to be_nil
16
+ end
17
+ end
18
+ end
19
+
20
+ describe "JAVASTACKTRACEPART" do
21
+ let(:pattern) { 'JAVASTACKTRACEPART' }
22
+ let(:message) { ' at com.sample.stacktrace.StackTraceExample.aMethod(StackTraceExample.java:42)' }
23
+ it "matches" do
24
+ grok = grok_match(pattern, message, true)
25
+ expect(grok).to include({
26
+ "message"=>" at com.sample.stacktrace.StackTraceExample.aMethod(StackTraceExample.java:42)",
27
+ "method"=>"aMethod",
28
+ "class"=>"com.sample.stacktrace.StackTraceExample",
29
+ "file"=>"StackTraceExample.java",
30
+ "line"=>"42"
31
+ })
32
+ end
33
+
34
+ context 'generated file' do
35
+ let(:message) { ' at org.jruby.RubyMethod$INVOKER$i$call.call(RubyMethod$INVOKER$i$call.gen)' }
36
+ it "matches" do
37
+ grok = grok_match(pattern, message, true)
38
+ expect(grok).to include({
39
+ "method"=>"call",
40
+ "class"=>"org.jruby.RubyMethod$INVOKER$i$call",
41
+ "file"=>"RubyMethod$INVOKER$i$call.gen",
42
+ })
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,61 @@
1
+ # encoding: utf-8
2
+ require "spec_helper"
3
+ require "logstash/patterns/core"
4
+
5
+ describe "MAVEN_VERSION" do
6
+
7
+ let(:pattern) { 'MAVEN_VERSION' }
8
+
9
+ context "when maven version is simple" do
10
+ let(:value) { '1.1.0' }
11
+
12
+ it "should match the version" do
13
+ expect(grok_match(pattern,value)).to pass
14
+ end
15
+ end
16
+
17
+ context "when maven version is a bit more complex" do
18
+ let(:value) { '2.35.128' }
19
+
20
+ it "should match the version" do
21
+ expect(grok_match(pattern,value)).to pass
22
+ end
23
+ end
24
+
25
+ context "when maven version contains release" do
26
+ let(:value) { '1.1.0.RELEASE' }
27
+
28
+ it "should match the version" do
29
+ expect(grok_match(pattern,value)).to pass
30
+ end
31
+ end
32
+
33
+ context "when maven version contains shapshot" do
34
+ let(:value) { '1.1.0.SNAPSHOT' }
35
+
36
+ it "should match the version" do
37
+ expect(grok_match(pattern,value)).to pass
38
+ end
39
+ end
40
+
41
+ context "when maven version contains release" do
42
+ context "and the version contains a dash" do
43
+ let(:value) { '1.1.0-RELEASE' }
44
+
45
+ it "should match the version" do
46
+ expect(grok_match(pattern,value)).to pass
47
+ end
48
+ end
49
+ end
50
+
51
+ context "when maven version contains shapshot" do
52
+ context "and the version contains a dash" do
53
+ let(:value) { '1.1.0-SNAPSHOT' }
54
+
55
+ it "should match the version" do
56
+ expect(grok_match(pattern,value)).to pass
57
+ end
58
+ end
59
+ end
60
+
61
+ end
@@ -82,7 +82,7 @@ end
82
82
 
83
83
  describe "NAGIOSLOGLINE - TIMEPERIOD TRANSITION" do
84
84
 
85
- let(:value) { "[1427925600] TIMEPERIOD TRANSITION: 24X7;1;1" }
85
+ let(:value) { "[1427925600] TIMEPERIOD TRANSITION: 24X7;-1;1" }
86
86
  let(:grok) { grok_match(subject, value) }
87
87
 
88
88
  it "a pattern pass the grok expression" do
@@ -105,6 +105,10 @@ describe "NAGIOSLOGLINE - TIMEPERIOD TRANSITION" do
105
105
  expect(grok).to include("nagios_service" => "24X7")
106
106
  end
107
107
 
108
+ it "generates the period from/to fields" do
109
+ expect(grok).to include("nagios_unknown1" => "-1", "nagios_unknown2" => "1")
110
+ end
111
+
108
112
  # Regression test for but fixed in Nagios patterns #30
109
113
  it "doesn't end in a semi-colon" do
110
114
  expect(grok['message']).to_not end_with(";")
@@ -0,0 +1,171 @@
1
+ # encoding: utf-8
2
+ require "spec_helper"
3
+ require "logstash/patterns/core"
4
+
5
+ describe "REDISTIMESTAMP" do
6
+
7
+ let(:value) { '14 Nov 07:01:22.119'}
8
+ let(:pattern) { "REDISTIMESTAMP" }
9
+
10
+ it "a pattern pass the grok expression" do
11
+ expect(grok_match(pattern, value)).to pass
12
+ end
13
+
14
+ end
15
+
16
+ describe "REDISLOG" do
17
+
18
+ let(:value) { "[4018] 14 Nov 07:01:22.119 * Background saving terminated with success" }
19
+ let(:pattern) { "REDISLOG" }
20
+ let(:grok) { grok_match(pattern, value) }
21
+
22
+ it "a pattern pass the grok expression" do
23
+ expect(grok).to pass
24
+ end
25
+
26
+ it "generates the pid field" do
27
+ expect(grok).to include("pid" => "4018")
28
+ end
29
+
30
+ end
31
+
32
+
33
+ describe "REDISMONLOG - SIMPLE COMMAND" do
34
+
35
+ let(:value) { "1470637867.953466 [0 195.168.1.1:52500] \"info\"" }
36
+ let(:pattern) { "REDISMONLOG" }
37
+ let(:grok) { grok_match(pattern, value) }
38
+
39
+ it "a pattern pass the grok expression" do
40
+ expect(grok).to pass
41
+ end
42
+
43
+ it "generates the timestamp field" do
44
+ expect(grok).to include("timestamp" => "1470637867.953466")
45
+ end
46
+
47
+ it "generates the database field" do
48
+ expect(grok).to include("database" => "0")
49
+ end
50
+
51
+ it "generates the client field" do
52
+ expect(grok).to include("client" => "195.168.1.1")
53
+ end
54
+
55
+ it "generates the port field" do
56
+ expect(grok).to include("port" => "52500")
57
+ end
58
+
59
+ it "generates the command field" do
60
+ expect(grok).to include("command" => "info")
61
+ end
62
+
63
+ end
64
+
65
+ describe "REDISMONLOG - ONE PARAM COMMAND" do
66
+
67
+ let(:value) { "1339518083.107412 [0 127.0.0.1:60866] \"keys\" \"*\"" }
68
+ let(:pattern) { "REDISMONLOG" }
69
+ let(:grok) { grok_match(pattern, value) }
70
+
71
+ it "a pattern pass the grok expression" do
72
+ expect(grok).to pass
73
+ end
74
+
75
+ it "generates the timestamp field" do
76
+ expect(grok).to include("timestamp" => "1339518083.107412")
77
+ end
78
+
79
+ it "generates the database field" do
80
+ expect(grok).to include("database" => "0")
81
+ end
82
+
83
+ it "generates the client field" do
84
+ expect(grok).to include("client" => "127.0.0.1")
85
+ end
86
+
87
+ it "generates the port field" do
88
+ expect(grok).to include("port" => "60866")
89
+ end
90
+
91
+ it "generates the command field" do
92
+ expect(grok).to include("command" => "keys")
93
+ end
94
+
95
+ it "generates the params field" do
96
+ expect(grok).to include("params" => "\"*\"")
97
+ end
98
+
99
+ end
100
+
101
+ describe "REDISMONLOG - TWO PARAM COMMAND" do
102
+
103
+ let(:value) { "1470637925.186681 [0 127.0.0.1:39404] \"rpush\" \"my:special:key\" \"{\\\"data\\\":\"cdr\\\",\\\"payload\\\":\\\"json\\\"}\"" }
104
+ let(:pattern) { "REDISMONLOG" }
105
+ let(:grok) { grok_match(pattern, value) }
106
+
107
+ it "a pattern pass the grok expression" do
108
+ expect(grok).to pass
109
+ end
110
+
111
+ it "generates the timestamp field" do
112
+ expect(grok).to include("timestamp" => "1470637925.186681")
113
+ end
114
+
115
+ it "generates the database field" do
116
+ expect(grok).to include("database" => "0")
117
+ end
118
+
119
+ it "generates the client field" do
120
+ expect(grok).to include("client" => "127.0.0.1")
121
+ end
122
+
123
+ it "generates the port field" do
124
+ expect(grok).to include("port" => "39404")
125
+ end
126
+
127
+ it "generates the command field" do
128
+ expect(grok).to include("command" => "rpush")
129
+ end
130
+
131
+ it "generates the params field" do
132
+ expect(grok).to include("params" => "\"my:special:key\" \"{\\\"data\\\":\"cdr\\\",\\\"payload\\\":\\\"json\\\"}\"")
133
+ end
134
+
135
+ end
136
+
137
+ describe "REDISMONLOG - VARIADIC COMMAND" do
138
+
139
+ let(:value) { "1470637875.777457 [15 195.168.1.1:52500] \"intentionally\" \"broken\" \"variadic\" \"log\" \"entry\"" }
140
+ let(:pattern) { "REDISMONLOG" }
141
+ let(:grok) { grok_match(pattern, value) }
142
+
143
+ it "a pattern pass the grok expression" do
144
+ expect(grok).to pass
145
+ end
146
+
147
+ it "generates the timestamp field" do
148
+ expect(grok).to include("timestamp" => "1470637875.777457")
149
+ end
150
+
151
+ it "generates the database field" do
152
+ expect(grok).to include("database" => "15")
153
+ end
154
+
155
+ it "generates the client field" do
156
+ expect(grok).to include("client" => "195.168.1.1")
157
+ end
158
+
159
+ it "generates the port field" do
160
+ expect(grok).to include("port" => "52500")
161
+ end
162
+
163
+ it "generates the command field" do
164
+ expect(grok).to include("command" => "intentionally")
165
+ end
166
+
167
+ it "generates the params field" do
168
+ expect(grok).to include("params" => "\"broken\" \"variadic\" \"log\" \"entry\"")
169
+ end
170
+
171
+ end