fluent-plugin-netflow 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +45 -0
- data/VERSION +1 -1
- data/lib/fluent/plugin/netflow_fields.yaml +21 -4
- data/lib/fluent/plugin/parser_netflow.rb +21 -25
- data/test/dump/netflow.v9.mpls-data.dump +0 -0
- data/test/dump/netflow.v9.mpls-template.dump +0 -0
- data/test/dump/netflow.v9.template.as2.dump +0 -0
- data/test/test_parser_netflow9.rb +40 -0
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5d48e590e8c0d4358dbec1e7fb8613cd828a2eff
|
4
|
+
data.tar.gz: 9245b86b319963bf4ea64573028003ac195b7ebe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e18722b820dfc5d0e28846ee42722407d3cd1d276758e9930f1305ca5fb06e23bf2257b715dd6e17111e33c9319957d6f86e5843fb9140747afb66b0b1da3cb9
|
7
|
+
data.tar.gz: 9b6a67536069dd3f9f17f76fecd384f91b1d24ba403ff09838ea28480069246075b0a3f41ebb1a8404da7f20c27a95907b11cb0fa71333179dc539758acdec07
|
data/README.md
CHANGED
@@ -109,6 +109,51 @@ end
|
|
109
109
|
**NOTE:**
|
110
110
|
If the plugin receives Netflow v9 from multiple sources, provide ```source_ip_address``` argument to parse correctly.
|
111
111
|
|
112
|
+
### Field definition for Netflow v9
|
113
|
+
|
114
|
+
Both option and scope fields for Netflow v9 are defined in [YAML](https://www.ietf.org/rfc/rfc3954.txt) where two parameters are described for each field value like:
|
115
|
+
|
116
|
+
```yaml
|
117
|
+
option:
|
118
|
+
...
|
119
|
+
4: # field value
|
120
|
+
- :uint8 # field length
|
121
|
+
- :protocol # field type
|
122
|
+
```
|
123
|
+
|
124
|
+
See [RFC3954 document](https://www.ietf.org/rfc/rfc3954.txt) for more details.
|
125
|
+
|
126
|
+
When int value specified for field length, the template parser in this plugin will prefer a field length in received template flowset over YAML. The int value in YAML will be used as a default value only when the length in received flowset is invalid.
|
127
|
+
|
128
|
+
```yaml
|
129
|
+
option:
|
130
|
+
1:
|
131
|
+
- 4 # means :unit32, which is just a default
|
132
|
+
- :in_bytes
|
133
|
+
```
|
134
|
+
|
135
|
+
When ```:skip``` is described for a field, the template parser will learn the length from received template flowset and skip the field when data flowsets are processed.
|
136
|
+
|
137
|
+
```yaml
|
138
|
+
option:
|
139
|
+
...
|
140
|
+
43:
|
141
|
+
- :skip
|
142
|
+
```
|
143
|
+
|
144
|
+
**NOTE:**
|
145
|
+
The definitions don't exactly reflect RFC3954 in order to cover some illegal implementations which export Netflow v9 in bad field length.
|
146
|
+
|
147
|
+
```yaml
|
148
|
+
31:
|
149
|
+
- 3 # Some system exports in 4 bytes despite of RFC
|
150
|
+
- :ipv6_flow_label
|
151
|
+
...
|
152
|
+
48:
|
153
|
+
- 1 # Some system exports in 2 bytes despite of RFC
|
154
|
+
- :flow_sampler_id
|
155
|
+
```
|
156
|
+
|
112
157
|
### More speed ?
|
113
158
|
|
114
159
|
:bullettrain_side: Try ```switched_times_from_uptime true``` option !
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.2
|
@@ -209,6 +209,27 @@ option:
|
|
209
209
|
72:
|
210
210
|
- :mpls_label
|
211
211
|
- :mpls_label_3
|
212
|
+
73:
|
213
|
+
- :mpls_label
|
214
|
+
- :mpls_label_4
|
215
|
+
74:
|
216
|
+
- :mpls_label
|
217
|
+
- :mpls_label_5
|
218
|
+
75:
|
219
|
+
- :mpls_label
|
220
|
+
- :mpls_label_6
|
221
|
+
76:
|
222
|
+
- :mpls_label
|
223
|
+
- :mpls_label_7
|
224
|
+
77:
|
225
|
+
- :mpls_label
|
226
|
+
- :mpls_label_8
|
227
|
+
78:
|
228
|
+
- :mpls_label
|
229
|
+
- :mpls_label_9
|
230
|
+
79:
|
231
|
+
- :mpls_label
|
232
|
+
- :mpls_label_10
|
212
233
|
80:
|
213
234
|
- :mac_addr
|
214
235
|
- :in_dst_mac
|
@@ -246,13 +267,9 @@ scope:
|
|
246
267
|
- :system
|
247
268
|
2:
|
248
269
|
- :skip
|
249
|
-
- :interface
|
250
270
|
3:
|
251
271
|
- :skip
|
252
|
-
- :line_card
|
253
272
|
4:
|
254
273
|
- :skip
|
255
|
-
- :netflow_cache
|
256
274
|
5:
|
257
275
|
- :skip
|
258
|
-
- :template
|
@@ -38,7 +38,7 @@ module Fluent
|
|
38
38
|
|
39
39
|
# Allow the user to augment/override/rename the supported Netflow fields
|
40
40
|
if @definitions
|
41
|
-
raise ConfigError, "definitions file #{@definitions}
|
41
|
+
raise ConfigError, "definitions file #{@definitions} doesn't exist" unless File.exist?(@definitions)
|
42
42
|
begin
|
43
43
|
@fields['option'].merge!(YAML.load_file(@definitions))
|
44
44
|
rescue => e
|
@@ -193,7 +193,7 @@ module Fluent
|
|
193
193
|
when 256..65535
|
194
194
|
handle_v9_flowset_data(host, pdu, flowset, block)
|
195
195
|
else
|
196
|
-
$log.warn
|
196
|
+
$log.warn 'Unsupported flowset', flowset_id: flowset.flowset_id
|
197
197
|
end
|
198
198
|
end
|
199
199
|
end
|
@@ -248,7 +248,8 @@ module Fluent
|
|
248
248
|
template_key = "#{host}|#{pdu.source_id}|#{flowset.flowset_id}"
|
249
249
|
template = @templates[template_key]
|
250
250
|
if ! template
|
251
|
-
$log.warn
|
251
|
+
$log.warn 'No matching template for',
|
252
|
+
host: host, source_id: pdu.source_id, flowset_id: flowset.flowset_id
|
252
253
|
return
|
253
254
|
end
|
254
255
|
|
@@ -306,31 +307,26 @@ module Fluent
|
|
306
307
|
end
|
307
308
|
|
308
309
|
def netflow_field_for(type, length, category='option')
|
309
|
-
|
310
|
-
field
|
311
|
-
|
312
|
-
|
313
|
-
if field[0].is_a?(Integer)
|
314
|
-
field[0] = uint_field(length, field[0])
|
315
|
-
end
|
310
|
+
unless field = @fields[category][type]
|
311
|
+
$log.warn "Skip unsupported field", type: type, length: length
|
312
|
+
return [:skip, nil, {length: length}]
|
313
|
+
end
|
316
314
|
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
field += [nil, {length: length}]
|
322
|
-
when :string
|
323
|
-
field += [{length: length, trim_padding: true}]
|
324
|
-
end
|
315
|
+
unless field.is_a?(Array)
|
316
|
+
$log.warn "Skip non-Array definition", field: field
|
317
|
+
return [:skip, nil, {length: length}]
|
318
|
+
end
|
325
319
|
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
320
|
+
# Small bit of fixup for numeric value, :skip or :string field length, which are dynamic
|
321
|
+
case field[0]
|
322
|
+
when Integer
|
323
|
+
[[uint_field(length, field[0]), field[1]]]
|
324
|
+
when :skip
|
325
|
+
[field + [nil, {length: length}]]
|
326
|
+
when :string
|
327
|
+
[field + [{length: length, trim_padding: true}]]
|
331
328
|
else
|
332
|
-
|
333
|
-
nil
|
329
|
+
[field]
|
334
330
|
end
|
335
331
|
end
|
336
332
|
|
Binary file
|
Binary file
|
Binary file
|
@@ -15,10 +15,18 @@ class Netflow9ParserTest < Test::Unit::TestCase
|
|
15
15
|
@raw_template ||= File.read(File.expand_path('../dump/netflow.v9.template.dump', __FILE__))
|
16
16
|
end
|
17
17
|
|
18
|
+
def raw_mpls_template
|
19
|
+
@raw_mpls_template ||= File.read(File.expand_path('../dump/netflow.v9.mpls-template.dump', __FILE__))
|
20
|
+
end
|
21
|
+
|
18
22
|
def raw_data
|
19
23
|
@raw_data ||= File.read(File.expand_path('../dump/netflow.v9.dump', __FILE__))
|
20
24
|
end
|
21
25
|
|
26
|
+
def raw_mpls_data
|
27
|
+
@raw_mpls_data ||= File.read(File.expand_path('../dump/netflow.v9.mpls-data.dump', __FILE__))
|
28
|
+
end
|
29
|
+
|
22
30
|
def raw_sampler_template
|
23
31
|
@raw_sampler_template ||= File.read(File.expand_path('../dump/netflow.v9.sampler_template.dump', __FILE__))
|
24
32
|
end
|
@@ -27,6 +35,10 @@ class Netflow9ParserTest < Test::Unit::TestCase
|
|
27
35
|
@raw_sampler_data ||= File.read(File.expand_path('../dump/netflow.v9.sampler.dump', __FILE__))
|
28
36
|
end
|
29
37
|
|
38
|
+
def raw_2byte_as_template
|
39
|
+
@raw_2byte_as_template ||= File.read(File.expand_path('../dump/netflow.v9.template.as2.dump', __FILE__))
|
40
|
+
end
|
41
|
+
|
30
42
|
DEFAULT_HOST = '127.0.0.1'
|
31
43
|
|
32
44
|
test 'parse netflow v9 binary data before loading corresponding template' do
|
@@ -127,4 +139,32 @@ class Netflow9ParserTest < Test::Unit::TestCase
|
|
127
139
|
assert_equal nil, parsed.first[1]['sampling_algorithm']
|
128
140
|
assert_equal nil, parsed.first[1]['sampling_interval']
|
129
141
|
end
|
142
|
+
|
143
|
+
test 'parse netflow v9 binary data with templates whose AS field length varies' do
|
144
|
+
parser = create_parser
|
145
|
+
|
146
|
+
parsed = []
|
147
|
+
[raw_2byte_as_template, raw_template].each {|raw| parser.call(raw, DEFAULT_HOST){} }
|
148
|
+
parser.call(raw_data, DEFAULT_HOST) do |time, record|
|
149
|
+
parsed << [time, record]
|
150
|
+
end
|
151
|
+
|
152
|
+
assert_equal 1, parsed.size
|
153
|
+
assert_equal 0, parsed.first[1]['src_as']
|
154
|
+
assert_equal 65000, parsed.first[1]['dst_as']
|
155
|
+
end
|
156
|
+
|
157
|
+
test 'parse netflow v9 binary data contains mpls information' do
|
158
|
+
parser = create_parser
|
159
|
+
|
160
|
+
parsed = []
|
161
|
+
[raw_sampler_template, raw_sampler_data, raw_mpls_template].each {|raw| parser.call(raw, DEFAULT_HOST){} }
|
162
|
+
parser.call(raw_mpls_data, DEFAULT_HOST) do |time, record|
|
163
|
+
parsed << [time, record]
|
164
|
+
end
|
165
|
+
|
166
|
+
assert_equal 24002, parsed.first[1]['mpls_label_1']
|
167
|
+
assert_equal '192.168.32.100', parsed.first[1]['ipv4_src_addr']
|
168
|
+
assert_equal '172.16.32.2', parsed.first[1]['ipv4_dst_addr']
|
169
|
+
end
|
130
170
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-netflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro Nakagawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-04-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -93,8 +93,11 @@ files:
|
|
93
93
|
- lib/fluent/plugin/vash.rb
|
94
94
|
- test/dump/netflow.v5.dump
|
95
95
|
- test/dump/netflow.v9.dump
|
96
|
+
- test/dump/netflow.v9.mpls-data.dump
|
97
|
+
- test/dump/netflow.v9.mpls-template.dump
|
96
98
|
- test/dump/netflow.v9.sampler.dump
|
97
99
|
- test/dump/netflow.v9.sampler_template.dump
|
100
|
+
- test/dump/netflow.v9.template.as2.dump
|
98
101
|
- test/dump/netflow.v9.template.dump
|
99
102
|
- test/helper.rb
|
100
103
|
- test/test_in_netflow.rb
|
@@ -127,8 +130,11 @@ summary: Netflow plugin for Fluentd
|
|
127
130
|
test_files:
|
128
131
|
- test/dump/netflow.v5.dump
|
129
132
|
- test/dump/netflow.v9.dump
|
133
|
+
- test/dump/netflow.v9.mpls-data.dump
|
134
|
+
- test/dump/netflow.v9.mpls-template.dump
|
130
135
|
- test/dump/netflow.v9.sampler.dump
|
131
136
|
- test/dump/netflow.v9.sampler_template.dump
|
137
|
+
- test/dump/netflow.v9.template.as2.dump
|
132
138
|
- test/dump/netflow.v9.template.dump
|
133
139
|
- test/helper.rb
|
134
140
|
- test/test_in_netflow.rb
|