logstash-patterns-core 4.2.0 → 4.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +98 -0
- data/Gemfile +3 -0
- data/README.md +11 -18
- data/lib/logstash/patterns/core.rb +11 -3
- data/logstash-patterns-core.gemspec +1 -1
- data/patterns/ecs-v1/aws +28 -0
- data/patterns/ecs-v1/bacula +53 -0
- data/patterns/ecs-v1/bind +13 -0
- data/patterns/ecs-v1/bro +30 -0
- data/patterns/ecs-v1/exim +26 -0
- data/patterns/ecs-v1/firewalls +111 -0
- data/patterns/ecs-v1/grok-patterns +95 -0
- data/patterns/ecs-v1/haproxy +40 -0
- data/patterns/ecs-v1/httpd +17 -0
- data/patterns/ecs-v1/java +34 -0
- data/patterns/ecs-v1/junos +13 -0
- data/patterns/ecs-v1/linux-syslog +16 -0
- data/patterns/{maven → ecs-v1/maven} +0 -0
- data/patterns/ecs-v1/mcollective +4 -0
- data/patterns/ecs-v1/mongodb +7 -0
- data/patterns/ecs-v1/nagios +124 -0
- data/patterns/ecs-v1/postgresql +2 -0
- data/patterns/ecs-v1/rails +13 -0
- data/patterns/ecs-v1/redis +3 -0
- data/patterns/ecs-v1/ruby +2 -0
- data/patterns/ecs-v1/squid +6 -0
- data/patterns/ecs-v1/zeek +33 -0
- data/patterns/{aws → legacy/aws} +1 -1
- data/patterns/{bacula → legacy/bacula} +5 -5
- data/patterns/legacy/bind +3 -0
- data/patterns/{bro → legacy/bro} +0 -0
- data/patterns/{exim → legacy/exim} +8 -2
- data/patterns/{firewalls → legacy/firewalls} +2 -2
- data/patterns/{grok-patterns → legacy/grok-patterns} +0 -0
- data/patterns/{haproxy → legacy/haproxy} +0 -0
- data/patterns/{httpd → legacy/httpd} +1 -1
- data/patterns/{java → legacy/java} +0 -0
- data/patterns/{junos → legacy/junos} +0 -0
- data/patterns/{linux-syslog → legacy/linux-syslog} +0 -0
- data/patterns/legacy/maven +1 -0
- data/patterns/{mcollective → legacy/mcollective} +0 -0
- data/patterns/{mcollective-patterns → legacy/mcollective-patterns} +0 -0
- data/patterns/{mongodb → legacy/mongodb} +0 -0
- data/patterns/{nagios → legacy/nagios} +0 -0
- data/patterns/{postgresql → legacy/postgresql} +0 -0
- data/patterns/{rails → legacy/rails} +0 -0
- data/patterns/{redis → legacy/redis} +0 -0
- data/patterns/{ruby → legacy/ruby} +0 -0
- data/patterns/legacy/squid +4 -0
- data/spec/patterns/aws_spec.rb +395 -0
- data/spec/patterns/bacula_spec.rb +367 -0
- data/spec/patterns/bind_spec.rb +78 -0
- data/spec/patterns/bro_spec.rb +613 -0
- data/spec/patterns/core_spec.rb +51 -9
- data/spec/patterns/exim_spec.rb +201 -0
- data/spec/patterns/firewalls_spec.rb +669 -66
- data/spec/patterns/haproxy_spec.rb +246 -38
- data/spec/patterns/httpd_spec.rb +215 -94
- data/spec/patterns/java_spec.rb +357 -27
- data/spec/patterns/junos_spec.rb +101 -0
- data/spec/patterns/mcollective_spec.rb +35 -0
- data/spec/patterns/mongodb_spec.rb +170 -33
- data/spec/patterns/nagios_spec.rb +296 -79
- data/spec/patterns/netscreen_spec.rb +123 -0
- data/spec/patterns/rails3_spec.rb +87 -29
- data/spec/patterns/redis_spec.rb +157 -121
- data/spec/patterns/shorewall_spec.rb +85 -74
- data/spec/patterns/squid_spec.rb +139 -0
- data/spec/patterns/syslog_spec.rb +266 -22
- data/spec/spec_helper.rb +80 -6
- metadata +64 -28
- data/patterns/bind +0 -3
- data/patterns/squid +0 -4
- data/spec/patterns/bro.rb +0 -126
- data/spec/patterns/s3_spec.rb +0 -173
@@ -2,89 +2,100 @@
|
|
2
2
|
require "spec_helper"
|
3
3
|
require "logstash/patterns/core"
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
let(:pattern) { "SHOREWALL" }
|
5
|
+
describe_pattern "SHOREWALL", ['legacy', 'ecs-v1'] do
|
8
6
|
|
9
7
|
context "parsing a message with OUT interface" do
|
10
8
|
|
11
|
-
let(:
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
9
|
+
let(:message) do
|
10
|
+
"May 28 17:23:25 myHost kernel: [3124658.791874] Shorewall:FORWARD:REJECT:" +
|
11
|
+
"IN=eth2 OUT=eth2 SRC=1.2.3.4 DST=192.168.0.10 LEN=141 TOS=0x00 PREC=0x00 TTL=63 ID=55251 PROTO=UDP SPT=5353 DPT=5335 LEN=121"
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'matches' do
|
15
|
+
expect(subject).to include("timestamp" => "May 28 17:23:25")
|
16
|
+
if ecs_compatibility?
|
17
|
+
expect(subject).to include(
|
18
|
+
"observer"=>{"hostname"=>"myHost", "ingress"=>{"interface"=>{"name"=>"eth2"}}, "egress"=>{"interface"=>{"name"=>"eth2"}}},
|
19
|
+
"shorewall"=>{'firewall'=>{"type"=>"FORWARD", "action"=>"REJECT"}},
|
20
|
+
"iptables"=>{
|
21
|
+
"length"=>141,
|
22
|
+
"tos"=>"00", "precedence_bits"=>"00",
|
23
|
+
"ttl"=>63,
|
24
|
+
"id"=>"55251"
|
25
|
+
},
|
26
|
+
"network"=>{"transport"=>"UDP"},
|
27
|
+
"source"=>{"ip"=>"1.2.3.4", "port"=>5353},
|
28
|
+
"destination"=>{"ip"=>"192.168.0.10", "port"=>5335}
|
29
|
+
)
|
30
|
+
else
|
31
|
+
expect(subject).to include("nf_host" => "myHost")
|
32
|
+
expect(subject).to include("nf_action1" => "FORWARD")
|
33
|
+
expect(subject).to include("nf_action2" => "REJECT")
|
34
|
+
expect(subject).to include("nf_in_interface" => "eth2")
|
35
|
+
expect(subject).to include("nf_out_interface" => "eth2")
|
36
|
+
expect(subject).to include("nf_src_ip" => "1.2.3.4")
|
37
|
+
expect(subject).to include("nf_dst_ip" => "192.168.0.10")
|
38
|
+
expect(subject).to include("nf_len" => "141")
|
39
|
+
expect(subject).to include("nf_tos" => "0x00")
|
40
|
+
expect(subject).to include("nf_prec" => "0x00")
|
41
|
+
expect(subject).to include("nf_ttl" => "63")
|
42
|
+
expect(subject).to include("nf_id" => "55251")
|
43
|
+
expect(subject).to include("nf_protocol" => "UDP")
|
44
|
+
expect(subject).to include("nf_src_port" => "5353")
|
45
|
+
expect(subject).to include("nf_dst_port" => "5335")
|
46
|
+
end
|
47
|
+
end
|
44
48
|
|
45
|
-
it { should include("nf_dst_port" => "5353") }
|
46
49
|
end
|
47
50
|
|
48
51
|
context "parsing a message without OUT interface" do
|
49
52
|
|
50
|
-
let(:value) { "May 28 17:31:07 myHost kernel: [3125121.106700] Shorewall:net2fw:DROP:IN=eth1 OUT= MAC=00:02:b3:c7:2f:77:38:72:c0:6e:92:9c:08:00 SRC=1.2.3.4 DST=1.2.3.4 LEN=60 TOS=0x00 PREC=0x00 TTL=49 ID=6480 DF PROTO=TCP SPT=59088 DPT=8080 WINDOW=2920 RES=0x00 SYN URGP=0" }
|
51
|
-
|
52
|
-
subject { grok_match(pattern, value) }
|
53
|
-
|
54
|
-
it { should include("timestamp" => "May 28 17:31:07") }
|
55
|
-
|
56
|
-
it { should include("nf_host" => "myHost") }
|
57
|
-
|
58
|
-
it { should include("nf_action1" => "net2fw") }
|
59
|
-
|
60
|
-
it { should include("nf_action2" => "DROP") }
|
61
|
-
|
62
|
-
it { should include("nf_in_interface" => "eth1") }
|
63
|
-
|
64
|
-
it { expect(subject["nf_out_interface"]).to be_nil }
|
65
|
-
|
66
|
-
it { should include("nf_dst_mac" => "00:02:b3:c7:2f:77") }
|
67
|
-
|
68
|
-
it { should include("nf_src_mac" => "38:72:c0:6e:92:9c") }
|
69
|
-
|
70
|
-
it { should include("nf_src_ip" => "1.2.3.4") }
|
71
|
-
|
72
|
-
it { should include("nf_dst_ip" => "1.2.3.4") }
|
73
|
-
|
74
|
-
it { should include("nf_len" => "60") }
|
75
|
-
|
76
|
-
it { should include("nf_tos" => "0x00") }
|
77
|
-
|
78
|
-
it { should include("nf_prec" => "0x00") }
|
79
|
-
|
80
|
-
it { should include("nf_ttl" => "49") }
|
81
|
-
|
82
|
-
it { should include("nf_id" => "6480") }
|
83
|
-
|
84
|
-
it { should include("nf_protocol" => "TCP") }
|
85
53
|
|
86
|
-
|
54
|
+
let(:message) do
|
55
|
+
"May 28 17:31:07 server Shorewall:net2fw:DROP:" +
|
56
|
+
"IN=eth1 OUT= MAC=00:02:b3:c7:2f:77:38:72:c0:6e:92:9c:08:00 SRC=127.0.0.1 DST=1.2.3.4 LEN=60 TOS=0x00 PREC=0x00 TTL=49 ID=6480 DF PROTO=TCP SPT=59088 DPT=8080 WINDOW=2920 RES=0x00 SYN URGP=0"
|
57
|
+
end
|
58
|
+
|
59
|
+
it 'matches' do
|
60
|
+
expect(subject).to include("timestamp" => "May 28 17:31:07")
|
61
|
+
if ecs_compatibility?
|
62
|
+
expect(subject).to include(
|
63
|
+
"observer"=>{"hostname"=>"server", "ingress"=>{"interface"=>{"name"=>"eth1"}}}, # no "output_interface"
|
64
|
+
"shorewall"=>{'firewall'=>{"type"=>"net2fw", "action"=>"DROP",}},
|
65
|
+
"iptables"=>{
|
66
|
+
"length"=>60,
|
67
|
+
"tos"=>"00", "precedence_bits"=>"00",
|
68
|
+
"ttl"=>49,
|
69
|
+
"id"=>"6480",
|
70
|
+
|
71
|
+
"fragment_flags"=>"DF",
|
72
|
+
"tcp"=>{"flags"=>"SYN ", "window"=>2920},
|
73
|
+
"tcp_reserved_bits"=>"00",
|
74
|
+
},
|
75
|
+
"network"=>{"transport"=>"TCP"}
|
76
|
+
)
|
77
|
+
expect(subject).to include("source"=>{"ip"=>"127.0.0.1", "port"=>59088, 'mac'=>"38:72:c0:6e:92:9c"})
|
78
|
+
expect(subject).to include("destination"=>{"ip"=>"1.2.3.4", "port"=>8080, 'mac'=>"00:02:b3:c7:2f:77"})
|
79
|
+
else
|
80
|
+
expect(subject).to include("nf_host" => "server")
|
81
|
+
expect(subject).to include("nf_action1" => "net2fw")
|
82
|
+
expect(subject).to include("nf_action2" => "DROP")
|
83
|
+
expect(subject).to include("nf_in_interface" => "eth1")
|
84
|
+
expect(subject["nf_out_interface"]).to be nil
|
85
|
+
expect(subject).to include("nf_dst_mac" => "00:02:b3:c7:2f:77")
|
86
|
+
expect(subject).to include("nf_src_mac" => "38:72:c0:6e:92:9c")
|
87
|
+
expect(subject).to include("nf_src_ip" => "127.0.0.1")
|
88
|
+
expect(subject).to include("nf_dst_ip" => "1.2.3.4")
|
89
|
+
expect(subject).to include("nf_len" => "60")
|
90
|
+
expect(subject).to include("nf_tos" => "0x00")
|
91
|
+
expect(subject).to include("nf_prec" => "0x00")
|
92
|
+
expect(subject).to include("nf_ttl" => "49")
|
93
|
+
expect(subject).to include("nf_id" => "6480")
|
94
|
+
expect(subject).to include("nf_protocol" => "TCP")
|
95
|
+
expect(subject).to include("nf_src_port" => "59088")
|
96
|
+
expect(subject).to include("nf_dst_port" => "8080")
|
97
|
+
end
|
98
|
+
end
|
87
99
|
|
88
|
-
it { should include("nf_dst_port" => "8080") }
|
89
100
|
end
|
90
101
|
end
|
@@ -0,0 +1,139 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "spec_helper"
|
3
|
+
require "logstash/patterns/core"
|
4
|
+
|
5
|
+
describe_pattern "SQUID3", ['legacy', 'ecs-v1'] do
|
6
|
+
|
7
|
+
describe 'CONNECT sample' do
|
8
|
+
|
9
|
+
let(:message) do
|
10
|
+
'1525344856.899 16867 10.170.72.111 TCP_TUNNEL/200 6256 CONNECT logs.ap-southeast-2.amazonaws.com:443 - HIER_DIRECT/53.140.206.134 -'
|
11
|
+
end
|
12
|
+
|
13
|
+
it "matches" do
|
14
|
+
expect(subject).to include("timestamp" => "1525344856.899")
|
15
|
+
if ecs_compatibility?
|
16
|
+
expect(subject).to include(
|
17
|
+
"event" => { "action" => "TCP_TUNNEL" },
|
18
|
+
"squid" => {
|
19
|
+
"request" => { "duration" => 16867 },
|
20
|
+
"hierarchy_code" => "HIER_DIRECT"
|
21
|
+
})
|
22
|
+
expect(subject).to include("destination" => { "address" => "53.140.206.134" })
|
23
|
+
expect(subject).to include("http" => {
|
24
|
+
"request" => { "method" => "CONNECT" },
|
25
|
+
"response" => { "bytes" => 6256, "status_code" => 200 }
|
26
|
+
# does not include missing ('-') as http.response.mime_type
|
27
|
+
})
|
28
|
+
expect(subject).to include("url" => { "original" => "logs.ap-southeast-2.amazonaws.com:443" })
|
29
|
+
expect(subject).to include("source" => { "ip" => "10.170.72.111" })
|
30
|
+
else
|
31
|
+
expect(subject).to include(
|
32
|
+
"duration" => "16867",
|
33
|
+
"client_address" => "10.170.72.111",
|
34
|
+
"cache_result" => "TCP_TUNNEL",
|
35
|
+
"status_code" => "200",
|
36
|
+
"request_method" => "CONNECT",
|
37
|
+
"bytes" => "6256",
|
38
|
+
"url" => "logs.ap-southeast-2.amazonaws.com:443",
|
39
|
+
"user" => "-",
|
40
|
+
"hierarchy_code" => "HIER_DIRECT",
|
41
|
+
"server" => "53.140.206.134",
|
42
|
+
"content_type" => "-",
|
43
|
+
)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
it "does not include missing ('-') user-name" do
|
48
|
+
if ecs_compatibility?
|
49
|
+
expect(subject.keys).to_not include("user") # "user" => { "name" => "-" }
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
describe 'GET sample' do
|
56
|
+
|
57
|
+
let(:message) do
|
58
|
+
"1525334330.556 3 120.65.1.1 TCP_REFRESH_MISS/200 2014 GET http://www.sample.com/hellow_world.txt public-user DIRECT/www.sample.com text/plain 902351708.872"
|
59
|
+
end
|
60
|
+
|
61
|
+
it "matches" do
|
62
|
+
expect(subject).to include("timestamp" => "1525334330.556")
|
63
|
+
if ecs_compatibility?
|
64
|
+
expect(subject).to include(
|
65
|
+
"event" => { "action" => "TCP_REFRESH_MISS" },
|
66
|
+
"squid" => {
|
67
|
+
"request" => { "duration" => 3 },
|
68
|
+
"hierarchy_code" => "DIRECT"
|
69
|
+
})
|
70
|
+
expect(subject).to include("destination" => { "address" => "www.sample.com" })
|
71
|
+
expect(subject).to include("http" => {
|
72
|
+
"request" => { "method" => "GET" },
|
73
|
+
"response" => { "bytes" => 2014, "status_code" => 200, "mime_type" => 'text/plain' }
|
74
|
+
})
|
75
|
+
expect(subject).to include("url" => { "original" => "http://www.sample.com/hellow_world.txt" })
|
76
|
+
expect(subject).to include("source" => { "ip" => "120.65.1.1" })
|
77
|
+
expect(subject).to include("user" => { "name" => "public-user" })
|
78
|
+
else
|
79
|
+
expect(subject).to include(
|
80
|
+
"duration"=>"3",
|
81
|
+
"client_address"=>"120.65.1.1",
|
82
|
+
"cache_result"=>"TCP_REFRESH_MISS",
|
83
|
+
"status_code"=>"200",
|
84
|
+
"bytes"=>"2014",
|
85
|
+
"request_method" => "GET",
|
86
|
+
"url" => "http://www.sample.com/hellow_world.txt",
|
87
|
+
"user"=>"public-user",
|
88
|
+
"hierarchy_code"=>"DIRECT",
|
89
|
+
"server"=>"www.sample.com",
|
90
|
+
"content_type"=>"text/plain",
|
91
|
+
)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
it "retains message" do
|
96
|
+
expect(subject).to include("message" => message)
|
97
|
+
end
|
98
|
+
|
99
|
+
end
|
100
|
+
|
101
|
+
context 'GET with invalid status' do # code 0 means unavailable - no response received
|
102
|
+
|
103
|
+
let(:message) do
|
104
|
+
'1426235912.366 16 192.168.3.50 TCP_MISS_ABORTED/000 0 GET http://garfield.com/comic/2015-03-13 - HIER_DIRECT/193.135.59.44 -'
|
105
|
+
end
|
106
|
+
|
107
|
+
it "matches" do
|
108
|
+
expect(subject).to include("timestamp" => "1426235912.366")
|
109
|
+
if ecs_compatibility?
|
110
|
+
expect(subject).to include("event"=>{"action"=>"TCP_MISS_ABORTED"},
|
111
|
+
"http"=>{"response"=>{"bytes"=>0}, "request"=>{"method"=>"GET"}})
|
112
|
+
else
|
113
|
+
expect(subject).to include("status_code"=>'000')
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
end
|
118
|
+
|
119
|
+
context 'GET with optional server ip' do
|
120
|
+
|
121
|
+
let(:message) do
|
122
|
+
'1066037222.352 132 144.157.100.17 TCP_MISS/504 1293 GET http://at.atremis.com/image/93101912/xyz - NONE/- -'
|
123
|
+
end
|
124
|
+
|
125
|
+
it "matches" do
|
126
|
+
expect(subject).to include("timestamp" => "1066037222.352")
|
127
|
+
if ecs_compatibility?
|
128
|
+
expect(subject).to include("event"=>{"action"=>"TCP_MISS"},
|
129
|
+
"http"=>{"response"=>{"bytes"=>1293, "status_code"=>504}, "request"=>{"method"=>"GET"}})
|
130
|
+
expect(subject.keys).not_to include('destination')
|
131
|
+
else
|
132
|
+
expect(subject).to include("status_code"=>'504')
|
133
|
+
expect(subject.keys).not_to include('server')
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
end
|
138
|
+
|
139
|
+
end
|
@@ -2,48 +2,292 @@
|
|
2
2
|
require "spec_helper"
|
3
3
|
require "logstash/patterns/core"
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
describe "SYSLOG5424BASE" do
|
8
|
-
it "matches host names in the syslog base pattern" do
|
9
|
-
expect(subject).to match("<174>1 2016-11-14T09:32:44+01:00 resolver.se named 6344 - - info: client 10.23.53.22#63252: query: googlehosted.l.googleusercontent.com IN A + (10.23.16.6)")
|
10
|
-
end
|
11
|
-
|
12
|
-
it "matches ipv4 in the syslog base pattern" do
|
13
|
-
expect(subject).to match("<174>1 2016-11-14T09:49:23+01:00 10.23.16.6 named 2255 - - info: client 10.23.56.93#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)")
|
14
|
-
end
|
15
|
-
|
16
|
-
it "matches ipv6 in the syslog base pattern" do
|
17
|
-
expect(subject).to match("<174>1 2016-11-14T09:49:23+01:00 2000:6a0:b:315:10:23:4:13 named 2255 - - info: client 10.23.56.9#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)")
|
18
|
-
end
|
19
|
-
end
|
5
|
+
describe_pattern "SYSLOGLINE", ['legacy', 'ecs-v1'] do
|
20
6
|
|
21
7
|
it "matches a simple message with pid" do
|
22
|
-
|
8
|
+
match = grok_match pattern, "May 11 15:17:02 meow.soy.se CRON[10973]: pam_unix(cron:session): session opened for user root by (uid=0)"
|
9
|
+
if ecs_compatibility?
|
10
|
+
expect(match).to include("process" => { "name" => "CRON", "pid" => 10973 })
|
11
|
+
else
|
12
|
+
expect(match).to include("pid" => "10973", "program" => "CRON")
|
13
|
+
end
|
23
14
|
end
|
24
15
|
|
25
16
|
it "matches prog with slash" do
|
26
|
-
|
17
|
+
match = grok_match pattern, "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]"
|
18
|
+
if ecs_compatibility?
|
19
|
+
expect(match).to include("process" => { "name" => "postfix/smtpd", "pid" => 1713 })
|
20
|
+
else
|
21
|
+
expect(match).to include("program" => "postfix/smtpd")
|
22
|
+
end
|
27
23
|
end
|
28
24
|
|
29
25
|
it "matches prog from ansible" do
|
30
|
-
|
26
|
+
message = "May 11 15:40:51 meow.soy.se ansible-<stdin>: Invoked with filter=* fact_path=/etc/ansible/facts.d"
|
27
|
+
match = grok_match pattern, message
|
28
|
+
if ecs_compatibility?
|
29
|
+
expect(match).to include(
|
30
|
+
"timestamp" => "May 11 15:40:51",
|
31
|
+
"process" => { "name" => "ansible-<stdin>" },
|
32
|
+
"host" => { "hostname" => "meow.soy.se" },
|
33
|
+
"message" => [message, "Invoked with filter=* fact_path=/etc/ansible/facts.d"]
|
34
|
+
)
|
35
|
+
else
|
36
|
+
expect(match).to include(
|
37
|
+
"timestamp" => "May 11 15:40:51",
|
38
|
+
"logsource" => "meow.soy.se",
|
39
|
+
"message" => [message, "Invoked with filter=* fact_path=/etc/ansible/facts.d"]
|
40
|
+
)
|
41
|
+
end
|
31
42
|
end
|
32
43
|
|
33
44
|
it "matches prog from RFC5424 APP-NAME" do
|
34
45
|
# https://tools.ietf.org/html/rfc5424#section-6.2.5
|
35
46
|
# https://tools.ietf.org/html/rfc5424#section-6
|
36
47
|
tag_from_rfc = ((33..126).map { |c| c.chr } - %w{[ ]}).join
|
37
|
-
|
48
|
+
match = grok_match pattern, "May 11 15:40:51 meow.soy.se #{tag_from_rfc}: Just some data which conforms to RFC5424"
|
49
|
+
if ecs_compatibility?
|
50
|
+
expect(match).to include("process" => { "name" => tag_from_rfc })
|
51
|
+
expect(match).to include("host" => { "hostname" => "meow.soy.se" })
|
52
|
+
else
|
53
|
+
expect(match).to include("logsource" => "meow.soy.se", "program" => tag_from_rfc)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'does not parse facility-level or msg-id' do
|
58
|
+
message = 'May 11 10:40:48 scrooge disk-health-nurse[26783]: [ID 702911 user.error] m:SY-mon-full-500 c:H : partition health measures for /var did not suffice - still using 96% of partition space'
|
59
|
+
match = grok_match pattern, message
|
60
|
+
expect(match).to include("timestamp" => "May 11 10:40:48")
|
61
|
+
expect(match).to include("message" => [message, "[ID 702911 user.error] m:SY-mon-full-500 c:H : partition health measures for /var did not suffice - still using 96% of partition space"])
|
62
|
+
if ecs_compatibility?
|
63
|
+
expect(match).to include("process"=>{"pid"=>26783, "name"=>"disk-health-nurse"}, "host"=>{"hostname"=>"scrooge"})
|
64
|
+
else
|
65
|
+
expect(match).to include("program"=>"disk-health-nurse", "pid"=>"26783", "logsource"=>"scrooge")
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
it 'parses (non-syslog) mesages without hostname' do
|
70
|
+
message = "Jan 11 22:33:44 su: 'su root' failed for luser on /dev/pts/8"
|
71
|
+
match = grok_match pattern, message
|
72
|
+
if ecs_compatibility? # in legacy mode a parse failure
|
73
|
+
expect(match).to include("process" => { "name" => "su" })
|
74
|
+
end
|
38
75
|
end
|
39
76
|
|
40
77
|
context "when having an optional progname" do
|
41
78
|
|
42
|
-
let(:
|
43
|
-
let(:value) { "<14>Jun 24 10:32:02 hostname WinFileService Event: read, Path: /.DS_Store, File/Folder: File, Size: 6.00 KB, User: user@host, IP: 123.123.123.123" }
|
79
|
+
let(:message) { "<14>Jun 24 10:32:02 win-host WinFileService Event: read, Path: /.DS_Store, File/Folder: File, Size: 6.00 KB, User: user@host, IP: 123.123.123.123" }
|
44
80
|
|
45
81
|
it "should accept the message" do
|
46
|
-
|
82
|
+
if ecs_compatibility?
|
83
|
+
expect(grok).to include("host" => { "hostname" => "win-host" })
|
84
|
+
else
|
85
|
+
expect(grok).to include("logsource" => "win-host")
|
86
|
+
end
|
87
|
+
expect(grok['message']).to eql [message, 'WinFileService Event: read, Path: /.DS_Store, File/Folder: File, Size: 6.00 KB, User: user@host, IP: 123.123.123.123']
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
describe_pattern "SYSLOG5424LINE", ['legacy', 'ecs-v1'] do
|
93
|
+
|
94
|
+
it "matches (ipv4 host)" do
|
95
|
+
message = "<174>1 2016-11-14T09:49:23+01:00 10.23.16.6 named 2255 - - info: client 10.23.56.93#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)"
|
96
|
+
match = grok_match pattern, message
|
97
|
+
if ecs_compatibility?
|
98
|
+
expect(match).to include("log" => { "syslog" => { "facility" => { "code" => 174 }}})
|
99
|
+
expect(match).to include("host" => { "hostname" => "10.23.16.6"})
|
100
|
+
expect(match).to include("process" => { "name" => "named", "pid" => 2255 })
|
101
|
+
expect(match).to include("system" => { "syslog" => { "version" => "1" }})
|
102
|
+
expect(match).to include("timestamp" => "2016-11-14T09:49:23+01:00")
|
103
|
+
expect(match).to include("message" => [message, "info: client 10.23.56.93#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)"])
|
104
|
+
else
|
105
|
+
expect(match).to include({
|
106
|
+
"syslog5424_pri" => "174",
|
107
|
+
"syslog5424_host" => "10.23.16.6",
|
108
|
+
"syslog5424_app" => "named",
|
109
|
+
"syslog5424_ver" => "1",
|
110
|
+
"syslog5424_proc" => "2255",
|
111
|
+
"syslog5424_ts" => "2016-11-14T09:49:23+01:00",
|
112
|
+
"syslog5424_msg" => "info: client 10.23.56.93#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)"
|
113
|
+
})
|
114
|
+
expect(match).to include("message" => message)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
it "matches ipv6 host" do
|
119
|
+
message = "<174>1 1995-04-12T23:20:50.52Z 2000:6a0:b:315:10:23:4:13 named 2255 - - info: client 10.23.56.9#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)"
|
120
|
+
match = grok_match pattern, message
|
121
|
+
if ecs_compatibility?
|
122
|
+
expect(match).to include("host" => { "hostname" => "2000:6a0:b:315:10:23:4:13" })
|
123
|
+
expect(match).to include("timestamp" => "1995-04-12T23:20:50.52Z")
|
124
|
+
expect(match).to include("message" => [message, "info: client 10.23.56.9#63295 (i1.tmg.com): query: i1.tmg.com IN A + (10.23.4.13)"])
|
125
|
+
else
|
126
|
+
expect(match).to include("syslog5424_host" => "2000:6a0:b:315:10:23:4:13")
|
127
|
+
expect(match).to include("syslog5424_ts" => "1995-04-12T23:20:50.52Z")
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
it "matches host name" do
|
132
|
+
message = "<174>1 2016-12-31T23:59:60-04:00 resolver.se prg00000[1234] - - info: client 10.23.53.22#63252: query: googlehosted.l.googleusercontent.com IN A + (10.23.16.6)"
|
133
|
+
match = grok_match pattern, message
|
134
|
+
if ecs_compatibility?
|
135
|
+
expect(match).to include("host" => { "hostname" => "resolver.se" })
|
136
|
+
expect(match).to include("message" => [message, "info: client 10.23.53.22#63252: query: googlehosted.l.googleusercontent.com IN A + (10.23.16.6)"])
|
137
|
+
expect(match).to include("timestamp" => "2016-12-31T23:59:60-04:00")
|
138
|
+
expect(match['system']['syslog'].keys).to_not include("message_id")
|
139
|
+
else
|
140
|
+
expect(match).to include("syslog5424_host" => "resolver.se")
|
141
|
+
expect(match).to include("syslog5424_ts" => "2016-12-31T23:59:60-04:00")
|
142
|
+
expect(match.keys).to_not include("syslog5424_msgid")
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
it "matches nil host-name" do
|
147
|
+
message = "<174>1 2003-08-24T05:14:15+00:00 - - - - - IN=enx503f560021dc OUT= MAC=01:00:5e:00:00:01:88:f8:c7:8b:1a:b4:04:00 SRC=192.168.81.1 DST=224.0.0.1 LEN=32 TOS=0x00 PREC=0xC0 TTL=1 ID=0 PROTO=2"
|
148
|
+
match = grok_match pattern, message
|
149
|
+
if ecs_compatibility?
|
150
|
+
expect(match).to include("timestamp" => "2003-08-24T05:14:15+00:00")
|
151
|
+
expect(match.keys).to_not include("host")
|
152
|
+
else
|
153
|
+
expect(match.keys).to_not include("syslog5424_host")
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
it 'matches milli-second precision timestamp' do
|
158
|
+
match = grok_match pattern, "<34>1 2003-08-24T05:14:15.123-07:00 the.borg.com su - ID47 - BOM'su root' failed for borg on /dev/pts/7"
|
159
|
+
if ecs_compatibility?
|
160
|
+
expect(match).to include("timestamp" => "2003-08-24T05:14:15.123-07:00")
|
161
|
+
expect(match).to include("process" => { "name" => "su" })
|
162
|
+
else
|
163
|
+
expect(match).to include("syslog5424_ts" => "2003-08-24T05:14:15.123-07:00")
|
164
|
+
end
|
165
|
+
end
|
166
|
+
|
167
|
+
it 'matches milli-second precision timestamp (Z) and msgid' do
|
168
|
+
match = grok_match pattern, "<34>1 2030-10-11T22:14:15.003Z the.borg.com su - m:WR-ID47 - BOM'su root' failed for borg on /dev/pts/7"
|
169
|
+
if ecs_compatibility?
|
170
|
+
expect(match).to include("timestamp" => "2030-10-11T22:14:15.003Z")
|
171
|
+
expect(match).to include("event" => { "code" => 'm:WR-ID47' })
|
172
|
+
else
|
173
|
+
expect(match).to include("syslog5424_ts" => "2030-10-11T22:14:15.003Z")
|
174
|
+
expect(match).to include("syslog5424_msgid" => "m:WR-ID47")
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
it 'matches micro-second precision timestamp and structured-data' do
|
179
|
+
message = "<34>1 2012-12-19T01:45:54.001234Z rufus - - - [meta sequenceId=10] BOMStack unit 3 Power supply 2 is down"
|
180
|
+
match = grok_match pattern, message
|
181
|
+
if ecs_compatibility?
|
182
|
+
expect(match).to include("timestamp" => "2012-12-19T01:45:54.001234Z")
|
183
|
+
expect(match).to include("system" => { "syslog" => hash_including("structured_data" => '[meta sequenceId=10]') })
|
184
|
+
expect(match).to include("message" => [message, "BOMStack unit 3 Power supply 2 is down"])
|
185
|
+
|
186
|
+
expect(match.keys).to_not include("process") # process.name
|
187
|
+
expect(match.keys).to_not include("event") # event.code
|
188
|
+
else
|
189
|
+
expect(match).to include("syslog5424_ts" => "2012-12-19T01:45:54.001234Z")
|
190
|
+
expect(match).to include("syslog5424_sd" => "[meta sequenceId=10]")
|
191
|
+
expect(match).to include("syslog5424_msg" => "BOMStack unit 3 Power supply 2 is down")
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
it 'matches more structured-data' do
|
196
|
+
sd = '[exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"][examplePriority@32473 class="high"]'
|
197
|
+
message = "<34>1 2012-12-19T01:45:54.001234+11:30 rufus - - - #{sd} [down]"
|
198
|
+
match = grok_match pattern, message
|
199
|
+
if ecs_compatibility?
|
200
|
+
expect(match).to include("system" => { "syslog" => hash_including("structured_data" => sd) })
|
201
|
+
expect(match).to include("message" => [message, "[down]"])
|
202
|
+
else
|
203
|
+
expect(match).to include("syslog5424_sd" => sd)
|
204
|
+
expect(match).to include("syslog5424_msg" => "[down]")
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
end
|
209
|
+
|
210
|
+
describe_pattern 'SYSLOGPAMSESSION', ['legacy', 'ecs-v1'] do
|
211
|
+
|
212
|
+
it "matches" do
|
213
|
+
message = 'Jul 14 13:36:03 precision pkexec: pam_unix(polkit-1:session): session opened for user root by (uid=1001)'
|
214
|
+
match = grok_match pattern, message
|
215
|
+
expect(match).to include("timestamp" => "Jul 14 13:36:03")
|
216
|
+
if ecs_compatibility?
|
217
|
+
expect(match).to include(
|
218
|
+
"host" => { "hostname" => "precision" },
|
219
|
+
"process" => { "name" => "pkexec" },
|
220
|
+
"user" => { "name" => "root" },
|
221
|
+
"system" => { "auth" => {
|
222
|
+
"pam" => { "module" => "pam_unix", "origin" => "polkit-1:session", "session_state" => "opened" }}
|
223
|
+
}
|
224
|
+
)
|
225
|
+
else
|
226
|
+
expect(match).to include({
|
227
|
+
"logsource"=>"precision",
|
228
|
+
"program"=>"pkexec",
|
229
|
+
"username"=>"root",
|
230
|
+
"pam_module"=>"pam_unix",
|
231
|
+
"pam_caller"=>"polkit-1:session",
|
232
|
+
"pam_session_state"=>"opened",
|
233
|
+
"pam_by"=>"(uid=1001)",
|
234
|
+
})
|
235
|
+
end
|
236
|
+
expect(match).to include("message" => [message, "pam_unix(polkit-1:session): session opened for user root by (uid=1001)"])
|
237
|
+
end
|
238
|
+
|
239
|
+
it "matches a message with pid" do
|
240
|
+
message = 'Jul 14 12:17:01.234 precision.computer CRON[869567]: pam_unix(cron:session): session closed for user root'
|
241
|
+
match = grok_match pattern, message
|
242
|
+
expect(match).to include("timestamp" => "Jul 14 12:17:01.234")
|
243
|
+
if ecs_compatibility?
|
244
|
+
expect(match).to include(
|
245
|
+
"host" => { "hostname" => "precision.computer" },
|
246
|
+
"process" => { "name" => "CRON", "pid" => 869567 },
|
247
|
+
"user" => { "name" => "root" },
|
248
|
+
"system" => { "auth" => {
|
249
|
+
"pam" => { "module" => "pam_unix", "origin" => "cron:session", "session_state" => "closed" }}
|
250
|
+
}
|
251
|
+
)
|
252
|
+
else
|
253
|
+
expect(match).to include({
|
254
|
+
"logsource" => "precision.computer",
|
255
|
+
"program" => "CRON",
|
256
|
+
"username" => "root",
|
257
|
+
"pid" => "869567",
|
258
|
+
"pam_module" => "pam_unix",
|
259
|
+
"pam_caller" => "cron:session",
|
260
|
+
"pam_session_state" => "closed"
|
261
|
+
})
|
47
262
|
end
|
263
|
+
expect(match).to include("message" => [message, "pam_unix(cron:session): session closed for user root"])
|
48
264
|
end
|
265
|
+
|
49
266
|
end
|
267
|
+
|
268
|
+
describe_pattern 'CRONLOG', ['legacy', 'ecs-v1'] do
|
269
|
+
|
270
|
+
it "matches" do
|
271
|
+
message = 'Jul 17 12:17:01 precision CRON[869568]: (root) CMD ( cd / && run-parts --report /etc/cron.hourly)'
|
272
|
+
match = grok_match pattern, message
|
273
|
+
expect(match).to include("timestamp" => "Jul 17 12:17:01")
|
274
|
+
if ecs_compatibility?
|
275
|
+
expect(match).to include(
|
276
|
+
"host" => { "hostname" => "precision" },
|
277
|
+
"process" => { "name" => "CRON", "pid" => 869568 },
|
278
|
+
"user" => { "name" => "root" },
|
279
|
+
"system" => { "cron" => { "action" => "CMD" } }
|
280
|
+
)
|
281
|
+
else
|
282
|
+
expect(match).to include(
|
283
|
+
"logsource"=>"precision",
|
284
|
+
"program"=>"CRON",
|
285
|
+
"pid"=>"869568",
|
286
|
+
"user"=>"root",
|
287
|
+
"action"=>"CMD"
|
288
|
+
)
|
289
|
+
end
|
290
|
+
expect(match).to include("message"=>[message, " cd / && run-parts --report /etc/cron.hourly"])
|
291
|
+
end
|
292
|
+
|
293
|
+
end
|