unified2 0.5.4 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/ChangeLog.md +10 -0
- data/README.md +41 -35
- data/Rakefile +3 -2
- data/bin/ru2 +76 -0
- data/example/example.rb +10 -18
- data/example/example2.rb +44 -0
- data/example/seeds/classification.config +1 -1
- data/example/seeds/gen-msg.map +86 -9
- data/example/seeds/sid-msg.map +2849 -316
- data/example/seeds/unified2-current.log +0 -0
- data/example/seeds/{unified2.log → unified2-legacy.log} +0 -0
- data/gemspec.yml +2 -1
- data/lib/unified2/classification.rb +12 -0
- data/lib/unified2/config_file.rb +4 -1
- data/lib/unified2/constructor/construct.rb +52 -6
- data/lib/unified2/constructor/event_ip4.rb +18 -3
- data/lib/unified2/constructor/event_ip6.rb +22 -4
- data/lib/unified2/constructor/extra_construct.rb +46 -0
- data/lib/unified2/constructor/extra_data.rb +37 -0
- data/lib/unified2/constructor/extra_data_header.rb +28 -0
- data/lib/unified2/constructor/legacy_event_ip4.rb +54 -0
- data/lib/unified2/constructor/legacy_event_ip6.rb +52 -0
- data/lib/unified2/constructor/packet.rb +9 -1
- data/lib/unified2/constructor/primitive/ipv4.rb +9 -0
- data/lib/unified2/constructor/record_header.rb +9 -0
- data/lib/unified2/constructor.rb +2 -1
- data/lib/unified2/core_ext/string.rb +2 -1
- data/lib/unified2/event.rb +290 -165
- data/lib/unified2/exceptions/binary_read_error.rb +11 -0
- data/lib/unified2/exceptions/file_not_found.rb +4 -1
- data/lib/unified2/exceptions/file_not_readable.rb +4 -1
- data/lib/unified2/exceptions/unknown_load_type.rb +4 -1
- data/lib/unified2/exceptions.rb +2 -1
- data/lib/unified2/extra.rb +128 -0
- data/lib/unified2/packet.rb +211 -0
- data/lib/unified2/protocol.rb +54 -63
- data/lib/unified2/sensor.rb +14 -2
- data/lib/unified2/signature.rb +12 -0
- data/lib/unified2/version.rb +4 -1
- data/lib/unified2.rb +65 -81
- data/spec/event_spec.rb +40 -27
- data/spec/legacy_event_spec.rb +122 -0
- data/spec/spec_helper.rb +10 -21
- data/spec/unified2_spec.rb +3 -3
- metadata +124 -140
- data/lib/unified2/payload.rb +0 -114
data/lib/unified2/exceptions.rb
CHANGED
@@ -0,0 +1,128 @@
|
|
1
|
+
#
|
2
|
+
# Unified2
|
3
|
+
#
|
4
|
+
module Unified2
|
5
|
+
|
6
|
+
#
|
7
|
+
# Extra
|
8
|
+
#
|
9
|
+
class Extra
|
10
|
+
|
11
|
+
# Type 1: True-Client-IP/XFF IPv4 address
|
12
|
+
# Type 2: True-Client-IP/XFF IPv6 address
|
13
|
+
# Type 3: ???
|
14
|
+
# Type 4: HTTP Gzip decompressed data
|
15
|
+
# Type 5: SMTP filename
|
16
|
+
# Type 6: SMTP MAIL FROM addresses
|
17
|
+
# Type 7: SMTP RCPT TO addresses
|
18
|
+
# Type 8: SMTP Email headers
|
19
|
+
# Type 9: HTTP Request URI
|
20
|
+
# Type 10: HTTP Request Hostname
|
21
|
+
# Type 11: Packet's IPv6 Source IP Address
|
22
|
+
# Type 12: Packet's IPv6 Destination IP Address
|
23
|
+
EXTRA_TYPES = {
|
24
|
+
1 => [
|
25
|
+
"EVENT_INFO_XFF_IPV4",
|
26
|
+
"True-Client-IP/XFF IPv4 address"
|
27
|
+
],
|
28
|
+
2 => [
|
29
|
+
"EVENT_INFO_XFF_IPV6",
|
30
|
+
"True-Client-IP/XFF IPv6 address"
|
31
|
+
],
|
32
|
+
3 => [
|
33
|
+
"EVENT_INFO_REVIEWED_BY",
|
34
|
+
"EVENT_INFO_REVIEWED_BY"
|
35
|
+
],
|
36
|
+
4 => [
|
37
|
+
"EVENT_INFO_GZIP_DATA",
|
38
|
+
"HTTP Gzip decompressed data"
|
39
|
+
],
|
40
|
+
5 => [
|
41
|
+
"EVENT_INFO_SMTP_FILENAME",
|
42
|
+
"SMTP filename"
|
43
|
+
],
|
44
|
+
6 => [
|
45
|
+
"EVENT_INFO_SMTP_MAILFROM",
|
46
|
+
"SMTP MAIL FROM addresses"
|
47
|
+
],
|
48
|
+
7 => [
|
49
|
+
"EVENT_INFO_SMTP_RCPTTO",
|
50
|
+
"SMTP RCPT TO addresses"
|
51
|
+
],
|
52
|
+
8 => [
|
53
|
+
"EVENT_INFO_SMTP_EMAIL_HDRS",
|
54
|
+
"SMTP Email headers"
|
55
|
+
],
|
56
|
+
9 => [
|
57
|
+
"EVENT_INFO_HTTP_URI",
|
58
|
+
"HTTP Request URI"
|
59
|
+
],
|
60
|
+
10 => [
|
61
|
+
"EVENT_INFO_HTTP_HOSTNAME",
|
62
|
+
"HTTP Request Hostname"
|
63
|
+
],
|
64
|
+
11 => [
|
65
|
+
"EVENT_INFO_IPV6_SRC",
|
66
|
+
"Packet's IPv6 Source IP Address"
|
67
|
+
],
|
68
|
+
12 => [
|
69
|
+
"EVENT_INFO_IPV6_DS",
|
70
|
+
"Packet's IPv6 Destination IP Addres"
|
71
|
+
]
|
72
|
+
}
|
73
|
+
|
74
|
+
#
|
75
|
+
# Build methods defaults
|
76
|
+
#
|
77
|
+
attr_reader :extra, :header, :type_id,
|
78
|
+
:data_type, :value, :length, :timestamp, :data
|
79
|
+
|
80
|
+
#
|
81
|
+
# Initialize Extra object
|
82
|
+
#
|
83
|
+
# @param [Hash] data Extra data hash
|
84
|
+
#
|
85
|
+
def initialize(data)
|
86
|
+
extra = data[:data]
|
87
|
+
@header = extra[:header]
|
88
|
+
@data = extra[:data]
|
89
|
+
|
90
|
+
@timestamp = Time.at(@data[:event_second].to_i)
|
91
|
+
@value = @data[:blob].to_s
|
92
|
+
@length = @data[:blob_length].to_i
|
93
|
+
@type_id = @data[:extra_type].to_i
|
94
|
+
@data_type = @data[:data_type].to_i
|
95
|
+
@type = EXTRA_TYPES[@type_id.to_i]
|
96
|
+
end
|
97
|
+
|
98
|
+
#
|
99
|
+
# Blank?
|
100
|
+
#
|
101
|
+
# @return [true, false] Check is extra value is blank
|
102
|
+
#
|
103
|
+
def blank?
|
104
|
+
return true unless @value
|
105
|
+
false
|
106
|
+
end
|
107
|
+
|
108
|
+
#
|
109
|
+
# Description
|
110
|
+
#
|
111
|
+
# @return [String] Extra data description
|
112
|
+
#
|
113
|
+
def description
|
114
|
+
@type.last
|
115
|
+
end
|
116
|
+
|
117
|
+
#
|
118
|
+
# Name
|
119
|
+
#
|
120
|
+
# @return [String] Extra data name
|
121
|
+
#
|
122
|
+
def name
|
123
|
+
@type.first
|
124
|
+
end
|
125
|
+
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
@@ -0,0 +1,211 @@
|
|
1
|
+
require 'hexdump'
|
2
|
+
require 'unified2/protocol'
|
3
|
+
|
4
|
+
#
|
5
|
+
# Unified2
|
6
|
+
#
|
7
|
+
module Unified2
|
8
|
+
|
9
|
+
|
10
|
+
#
|
11
|
+
# Packet
|
12
|
+
#
|
13
|
+
class Packet
|
14
|
+
|
15
|
+
#
|
16
|
+
# Build method defaults
|
17
|
+
#
|
18
|
+
attr_reader :link_type, :event_id,
|
19
|
+
:microsecond, :timestamp, :length,
|
20
|
+
:raw, :event_timestamp
|
21
|
+
|
22
|
+
#
|
23
|
+
# Initialize packet Object
|
24
|
+
#
|
25
|
+
# @param [Hash] Packet Packet hash
|
26
|
+
#
|
27
|
+
def initialize(packet)
|
28
|
+
@raw = packet
|
29
|
+
@link_type = packet[:linktype]
|
30
|
+
@microsecond = packet[:packet_microsecond]
|
31
|
+
|
32
|
+
@event_timestamp = Time.at(packet[:timestamp])
|
33
|
+
@timestamp = Time.at(packet[:packet_timestamp])
|
34
|
+
@length = packet[:packet_length].to_i
|
35
|
+
@event_id = packet[:event_id]
|
36
|
+
|
37
|
+
@packet ||= PacketFu::Packet.parse(packet[:packet])
|
38
|
+
@protocol = @packet.protocol.last.to_sym
|
39
|
+
end
|
40
|
+
|
41
|
+
#
|
42
|
+
# IP Header
|
43
|
+
#
|
44
|
+
# @return [Hash] IP header
|
45
|
+
#
|
46
|
+
def ip_header
|
47
|
+
if @packet.is_ip?
|
48
|
+
@ip_header = {
|
49
|
+
:ip_ver => @packet.ip_header.ip_v,
|
50
|
+
:ip_hlen => @packet.ip_header.ip_hl,
|
51
|
+
:ip_tos => @packet.ip_header.ip_tos,
|
52
|
+
:ip_len => @packet.ip_header.ip_len,
|
53
|
+
:ip_id => @packet.ip_header.ip_id,
|
54
|
+
:ip_frag => @packet.ip_header.ip_frag,
|
55
|
+
:ip_ttl => @packet.ip_header.ip_ttl,
|
56
|
+
:ip_proto => @packet.ip_header.ip_proto,
|
57
|
+
:ip_csum => @packet.ip_header.ip_sum
|
58
|
+
}
|
59
|
+
else
|
60
|
+
@ip_header = {}
|
61
|
+
end
|
62
|
+
|
63
|
+
@ip_header
|
64
|
+
end
|
65
|
+
|
66
|
+
#
|
67
|
+
# Protocol
|
68
|
+
#
|
69
|
+
# @return [Protocol] packet protocol object
|
70
|
+
#
|
71
|
+
def protocol
|
72
|
+
@proto ||= Protocol.new(@protocol, @packet)
|
73
|
+
end
|
74
|
+
|
75
|
+
#
|
76
|
+
# String
|
77
|
+
#
|
78
|
+
# @return [String] Signature name
|
79
|
+
#
|
80
|
+
def to_s
|
81
|
+
payload.to_s
|
82
|
+
end
|
83
|
+
|
84
|
+
#
|
85
|
+
# Payload
|
86
|
+
#
|
87
|
+
# @return [Payload] Event payload object
|
88
|
+
#
|
89
|
+
def payload
|
90
|
+
@packet.payload
|
91
|
+
end
|
92
|
+
|
93
|
+
#
|
94
|
+
# Blank?
|
95
|
+
#
|
96
|
+
# @return [true, false] Check is payload is blank
|
97
|
+
#
|
98
|
+
def blank?
|
99
|
+
return true unless @packet
|
100
|
+
false
|
101
|
+
end
|
102
|
+
|
103
|
+
#
|
104
|
+
# Raw
|
105
|
+
#
|
106
|
+
# @return [String] Raw binary payload
|
107
|
+
#
|
108
|
+
def raw
|
109
|
+
@packet
|
110
|
+
end
|
111
|
+
|
112
|
+
#
|
113
|
+
# Hex
|
114
|
+
#
|
115
|
+
# @return [String] Convert payload to hex
|
116
|
+
#
|
117
|
+
def hex(include_header=true)
|
118
|
+
packet = if include_header
|
119
|
+
@packet.to_s
|
120
|
+
else
|
121
|
+
@packet.payload.to_s
|
122
|
+
end
|
123
|
+
|
124
|
+
hex = packet.unpack('H*')
|
125
|
+
return hex.first if hex
|
126
|
+
nil
|
127
|
+
end
|
128
|
+
|
129
|
+
#
|
130
|
+
# Dump
|
131
|
+
#
|
132
|
+
# @param [options] options Hash of options for Hexdump#dump
|
133
|
+
#
|
134
|
+
# @option options [Integer] :width (16)
|
135
|
+
# The number of bytes to dump for each line.
|
136
|
+
#
|
137
|
+
# @option options [Symbol, Integer] :base (:hexadecimal)
|
138
|
+
# The base to print bytes in. Supported bases include, `:hexadecimal`,
|
139
|
+
# `:hex`, `16, `:decimal`, `:dec`, `10, `:octal`, `:oct`, `8`,
|
140
|
+
# `:binary`, `:bin` and `2`.
|
141
|
+
#
|
142
|
+
# @option options [Boolean] :ascii (false)
|
143
|
+
# Print ascii characters when possible.
|
144
|
+
#
|
145
|
+
# @option options [#<<] :output (STDOUT)
|
146
|
+
# The output to print the hexdump to.
|
147
|
+
#
|
148
|
+
# @yield [index,hex_segment,print_segment]
|
149
|
+
# The given block will be passed the hexdump break-down of each segment.
|
150
|
+
#
|
151
|
+
# @yieldparam [Integer] index
|
152
|
+
# The index of the hexdumped segment.
|
153
|
+
#
|
154
|
+
# @yieldparam [Array<String>] hex_segment
|
155
|
+
# The hexadecimal-byte representation of the segment.
|
156
|
+
#
|
157
|
+
# @yieldparam [Array<String>] print_segment
|
158
|
+
# The print-character representation of the segment.
|
159
|
+
#
|
160
|
+
# @return [nil]
|
161
|
+
#
|
162
|
+
# @raise [ArgumentError]
|
163
|
+
# The given data does not define the `#each_byte` method, or
|
164
|
+
#
|
165
|
+
# @note
|
166
|
+
# Please view the hexdump documentation for more
|
167
|
+
# information. Hexdump is a great lib by @postmodern.
|
168
|
+
# (http://github.com/postmodern/hexdump)
|
169
|
+
#
|
170
|
+
def dump(options={})
|
171
|
+
packet = if options[:header]
|
172
|
+
@raw[:packet]
|
173
|
+
else
|
174
|
+
@packet.payload
|
175
|
+
end
|
176
|
+
|
177
|
+
Hexdump.dump(packet, options)
|
178
|
+
end
|
179
|
+
|
180
|
+
#
|
181
|
+
# Hexdump
|
182
|
+
#
|
183
|
+
# @see Packet#dump
|
184
|
+
#
|
185
|
+
# @example
|
186
|
+
# packet.hexdump(:width => 16)
|
187
|
+
#
|
188
|
+
def hexdump(options={})
|
189
|
+
hexdump = options[:output] ||= ""
|
190
|
+
options[:width] ||= 30
|
191
|
+
options[:header] ||= true
|
192
|
+
|
193
|
+
dump(options)
|
194
|
+
hexdump
|
195
|
+
end
|
196
|
+
|
197
|
+
#
|
198
|
+
# Checksum
|
199
|
+
#
|
200
|
+
# Create a unique payload checksum
|
201
|
+
#
|
202
|
+
# @return [String] Payload checksum
|
203
|
+
#
|
204
|
+
def checksum
|
205
|
+
Digest::MD5.hexdigest(hex(false))
|
206
|
+
end
|
207
|
+
|
208
|
+
end # class Packet
|
209
|
+
|
210
|
+
end # module Unified2
|
211
|
+
|
data/lib/unified2/protocol.rb
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
#
|
2
|
+
# Unified2
|
3
|
+
#
|
1
4
|
module Unified2
|
2
5
|
#
|
3
6
|
# Protocol
|
@@ -16,29 +19,13 @@ module Unified2
|
|
16
19
|
@packet = packet
|
17
20
|
end
|
18
21
|
|
19
|
-
#
|
20
|
-
# Protocol Header
|
21
|
-
#
|
22
|
-
# @return [Object, nil] Protocol header object
|
23
|
-
#
|
24
|
-
def header
|
25
|
-
if @packet.has_data?
|
26
|
-
if @packet.send(:"is_#{@protocol.downcase}?")
|
27
|
-
@packet.send(:"#{@protocol.downcase}_header")
|
28
|
-
end
|
29
|
-
else
|
30
|
-
nil
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
22
|
#
|
35
23
|
# ICMP?
|
36
24
|
#
|
37
25
|
# @return [true, false] Check is protocol is icmp
|
38
26
|
#
|
39
27
|
def icmp?
|
40
|
-
|
41
|
-
false
|
28
|
+
@protocol == :ICMP
|
42
29
|
end
|
43
30
|
|
44
31
|
#
|
@@ -47,8 +34,7 @@ module Unified2
|
|
47
34
|
# @return [true, false] Check is protocol is tcp
|
48
35
|
#
|
49
36
|
def tcp?
|
50
|
-
|
51
|
-
false
|
37
|
+
@protocol == :TCP
|
52
38
|
end
|
53
39
|
|
54
40
|
#
|
@@ -57,8 +43,7 @@ module Unified2
|
|
57
43
|
# @return [true, false] Check is protocol is udp
|
58
44
|
#
|
59
45
|
def udp?
|
60
|
-
|
61
|
-
false
|
46
|
+
@protocol == :UDP
|
62
47
|
end
|
63
48
|
|
64
49
|
#
|
@@ -88,53 +73,59 @@ module Unified2
|
|
88
73
|
{}
|
89
74
|
end
|
90
75
|
end
|
76
|
+
alias header to_h
|
91
77
|
|
92
78
|
private
|
93
79
|
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
:csum => header.icmp_sum,
|
99
|
-
:code => header.icmp_code
|
100
|
-
}
|
101
|
-
|
102
|
-
@icmp[:body] = header.body if include_body
|
103
|
-
|
104
|
-
@icmp
|
105
|
-
end
|
80
|
+
def hdr
|
81
|
+
return nil unless @packet.send(:"is_#{@protocol.downcase}?")
|
82
|
+
@packet.send(:"#{@protocol.downcase}_header")
|
83
|
+
end
|
106
84
|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
85
|
+
def icmp(include_body=false)
|
86
|
+
icmp = {
|
87
|
+
:length => hdr.len,
|
88
|
+
:type => hdr.icmp_type,
|
89
|
+
:csum => hdr.icmp_sum,
|
90
|
+
:code => hdr.icmp_code
|
91
|
+
}
|
92
|
+
|
93
|
+
icmp[:body] = hdr.body if include_body
|
94
|
+
|
95
|
+
icmp
|
96
|
+
end
|
117
97
|
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
98
|
+
def udp(include_body=false)
|
99
|
+
udp = {
|
100
|
+
:length => hdr.len,
|
101
|
+
:csum => hdr.udp_sum,
|
102
|
+
}
|
103
|
+
|
104
|
+
udp[:body] = hdr.body if include_body
|
105
|
+
|
106
|
+
udp
|
107
|
+
end
|
108
|
+
|
109
|
+
def tcp(include_body=false)
|
110
|
+
tcp = {
|
111
|
+
:length => hdr.len,
|
112
|
+
:seq => hdr.tcp_seq,
|
113
|
+
:ack => hdr.tcp_ack,
|
114
|
+
:win => hdr.tcp_win,
|
115
|
+
:csum => hdr.tcp_sum,
|
116
|
+
:urg => hdr.tcp_urg,
|
117
|
+
:hlen => hdr.tcp_hlen,
|
118
|
+
:reserved => hdr.tcp_reserved,
|
119
|
+
:ecn => hdr.tcp_ecn,
|
120
|
+
:opts_len => hdr.tcp_opts_len,
|
121
|
+
:rand_port => hdr.rand_port,
|
122
|
+
:options => hdr.tcp_options
|
123
|
+
}
|
124
|
+
|
125
|
+
tcp[:body] = hdr.body if include_body
|
126
|
+
|
127
|
+
tcp
|
128
|
+
end
|
138
129
|
|
139
130
|
end
|
140
131
|
end
|
data/lib/unified2/sensor.rb
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
#
|
2
|
+
# Unified2
|
3
|
+
#
|
1
4
|
module Unified2
|
2
5
|
#
|
3
6
|
# Sensor
|
@@ -22,7 +25,16 @@ module Unified2
|
|
22
25
|
@interface ||= options[:interface] || nil
|
23
26
|
@checksum = nil
|
24
27
|
end
|
25
|
-
|
28
|
+
|
29
|
+
#
|
30
|
+
# To String
|
31
|
+
#
|
32
|
+
# @return [String] Sensor Name
|
33
|
+
#
|
34
|
+
def to_s
|
35
|
+
@name
|
36
|
+
end
|
37
|
+
|
26
38
|
#
|
27
39
|
# Update
|
28
40
|
#
|
@@ -45,4 +57,4 @@ module Unified2
|
|
45
57
|
end
|
46
58
|
|
47
59
|
end
|
48
|
-
end
|
60
|
+
end
|
data/lib/unified2/signature.rb
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
#
|
2
|
+
# Unified2
|
3
|
+
#
|
1
4
|
module Unified2
|
2
5
|
#
|
3
6
|
# Signature
|
@@ -25,6 +28,15 @@ module Unified2
|
|
25
28
|
@blank = signature[:blank] || false
|
26
29
|
end
|
27
30
|
|
31
|
+
#
|
32
|
+
# to_string
|
33
|
+
#
|
34
|
+
# @return [String] Signature name
|
35
|
+
#
|
36
|
+
def to_s
|
37
|
+
@name
|
38
|
+
end
|
39
|
+
|
28
40
|
#
|
29
41
|
# Blank?
|
30
42
|
#
|