fluent-plugin-netflow 0.2.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3cd8781b6ecc14e1dd982e4046b795a34412e768
4
- data.tar.gz: be9f57fbb0fd1323fba31f718146632e6d2d5fdc
3
+ metadata.gz: 5d48e590e8c0d4358dbec1e7fb8613cd828a2eff
4
+ data.tar.gz: 9245b86b319963bf4ea64573028003ac195b7ebe
5
5
  SHA512:
6
- metadata.gz: aca8f4e3c4f146ad05e0b0f166e1a7bd65e77352e4ce0927101e5d2d6b3f0d6ad443a39ba976c8c47b917366bf363ff2d1a17ed0bc0b068e4be81879b9d9f27e
7
- data.tar.gz: 14d3a412b02acccd57ef7723ee1b2b32e87f3b3c34d5f4e14f6b85bb5fc6f23f78d575c809a24a56e410879af6b6a22f139a62279a400340f3a827ebcc257f2b
6
+ metadata.gz: e18722b820dfc5d0e28846ee42722407d3cd1d276758e9930f1305ca5fb06e23bf2257b715dd6e17111e33c9319957d6f86e5843fb9140747afb66b0b1da3cb9
7
+ data.tar.gz: 9b6a67536069dd3f9f17f76fecd384f91b1d24ba403ff09838ea28480069246075b0a3f41ebb1a8404da7f20c27a95907b11cb0fa71333179dc539758acdec07
data/README.md CHANGED
@@ -109,6 +109,51 @@ end
109
109
  **NOTE:**
110
110
  If the plugin receives Netflow v9 from multiple sources, provide ```source_ip_address``` argument to parse correctly.
111
111
 
112
+ ### Field definition for Netflow v9
113
+
114
+ Both option and scope fields for Netflow v9 are defined in [YAML](https://www.ietf.org/rfc/rfc3954.txt) where two parameters are described for each field value like:
115
+
116
+ ```yaml
117
+ option:
118
+ ...
119
+ 4: # field value
120
+ - :uint8 # field length
121
+ - :protocol # field type
122
+ ```
123
+
124
+ See [RFC3954 document](https://www.ietf.org/rfc/rfc3954.txt) for more details.
125
+
126
+ When int value specified for field length, the template parser in this plugin will prefer a field length in received template flowset over YAML. The int value in YAML will be used as a default value only when the length in received flowset is invalid.
127
+
128
+ ```yaml
129
+ option:
130
+ 1:
131
+ - 4 # means :unit32, which is just a default
132
+ - :in_bytes
133
+ ```
134
+
135
+ When ```:skip``` is described for a field, the template parser will learn the length from received template flowset and skip the field when data flowsets are processed.
136
+
137
+ ```yaml
138
+ option:
139
+ ...
140
+ 43:
141
+ - :skip
142
+ ```
143
+
144
+ **NOTE:**
145
+ The definitions don't exactly reflect RFC3954 in order to cover some illegal implementations which export Netflow v9 in bad field length.
146
+
147
+ ```yaml
148
+ 31:
149
+ - 3 # Some system exports in 4 bytes despite of RFC
150
+ - :ipv6_flow_label
151
+ ...
152
+ 48:
153
+ - 1 # Some system exports in 2 bytes despite of RFC
154
+ - :flow_sampler_id
155
+ ```
156
+
112
157
  ### More speed ?
113
158
 
114
159
  :bullettrain_side: Try ```switched_times_from_uptime true``` option !
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.2.1
1
+ 0.2.2
@@ -209,6 +209,27 @@ option:
209
209
  72:
210
210
  - :mpls_label
211
211
  - :mpls_label_3
212
+ 73:
213
+ - :mpls_label
214
+ - :mpls_label_4
215
+ 74:
216
+ - :mpls_label
217
+ - :mpls_label_5
218
+ 75:
219
+ - :mpls_label
220
+ - :mpls_label_6
221
+ 76:
222
+ - :mpls_label
223
+ - :mpls_label_7
224
+ 77:
225
+ - :mpls_label
226
+ - :mpls_label_8
227
+ 78:
228
+ - :mpls_label
229
+ - :mpls_label_9
230
+ 79:
231
+ - :mpls_label
232
+ - :mpls_label_10
212
233
  80:
213
234
  - :mac_addr
214
235
  - :in_dst_mac
@@ -246,13 +267,9 @@ scope:
246
267
  - :system
247
268
  2:
248
269
  - :skip
249
- - :interface
250
270
  3:
251
271
  - :skip
252
- - :line_card
253
272
  4:
254
273
  - :skip
255
- - :netflow_cache
256
274
  5:
257
275
  - :skip
258
- - :template
@@ -38,7 +38,7 @@ module Fluent
38
38
 
39
39
  # Allow the user to augment/override/rename the supported Netflow fields
40
40
  if @definitions
41
- raise ConfigError, "definitions file #{@definitions} does not exists" unless File.exist?(@definitions)
41
+ raise ConfigError, "definitions file #{@definitions} doesn't exist" unless File.exist?(@definitions)
42
42
  begin
43
43
  @fields['option'].merge!(YAML.load_file(@definitions))
44
44
  rescue => e
@@ -193,7 +193,7 @@ module Fluent
193
193
  when 256..65535
194
194
  handle_v9_flowset_data(host, pdu, flowset, block)
195
195
  else
196
- $log.warn "Unsupported flowset id #{flowset.flowset_id}"
196
+ $log.warn 'Unsupported flowset', flowset_id: flowset.flowset_id
197
197
  end
198
198
  end
199
199
  end
@@ -248,7 +248,8 @@ module Fluent
248
248
  template_key = "#{host}|#{pdu.source_id}|#{flowset.flowset_id}"
249
249
  template = @templates[template_key]
250
250
  if ! template
251
- $log.warn("No matching template for flow id #{flowset.flowset_id}")
251
+ $log.warn 'No matching template for',
252
+ host: host, source_id: pdu.source_id, flowset_id: flowset.flowset_id
252
253
  return
253
254
  end
254
255
 
@@ -306,31 +307,26 @@ module Fluent
306
307
  end
307
308
 
308
309
  def netflow_field_for(type, length, category='option')
309
- if @fields[category].include?(type)
310
- field = @fields[category][type]
311
- if field.is_a?(Array)
312
-
313
- if field[0].is_a?(Integer)
314
- field[0] = uint_field(length, field[0])
315
- end
310
+ unless field = @fields[category][type]
311
+ $log.warn "Skip unsupported field", type: type, length: length
312
+ return [:skip, nil, {length: length}]
313
+ end
316
314
 
317
- # Small bit of fixup for skip or string field types where the length
318
- # is dynamic
319
- case field[0]
320
- when :skip
321
- field += [nil, {length: length}]
322
- when :string
323
- field += [{length: length, trim_padding: true}]
324
- end
315
+ unless field.is_a?(Array)
316
+ $log.warn "Skip non-Array definition", field: field
317
+ return [:skip, nil, {length: length}]
318
+ end
325
319
 
326
- [field]
327
- else
328
- $log.warn "Definition should be an array", field: field
329
- nil
330
- end
320
+ # Small bit of fixup for numeric value, :skip or :string field length, which are dynamic
321
+ case field[0]
322
+ when Integer
323
+ [[uint_field(length, field[0]), field[1]]]
324
+ when :skip
325
+ [field + [nil, {length: length}]]
326
+ when :string
327
+ [field + [{length: length, trim_padding: true}]]
331
328
  else
332
- $log.warn "Unsupported field", type: type, length: length
333
- nil
329
+ [field]
334
330
  end
335
331
  end
336
332
 
Binary file
@@ -15,10 +15,18 @@ class Netflow9ParserTest < Test::Unit::TestCase
15
15
  @raw_template ||= File.read(File.expand_path('../dump/netflow.v9.template.dump', __FILE__))
16
16
  end
17
17
 
18
+ def raw_mpls_template
19
+ @raw_mpls_template ||= File.read(File.expand_path('../dump/netflow.v9.mpls-template.dump', __FILE__))
20
+ end
21
+
18
22
  def raw_data
19
23
  @raw_data ||= File.read(File.expand_path('../dump/netflow.v9.dump', __FILE__))
20
24
  end
21
25
 
26
+ def raw_mpls_data
27
+ @raw_mpls_data ||= File.read(File.expand_path('../dump/netflow.v9.mpls-data.dump', __FILE__))
28
+ end
29
+
22
30
  def raw_sampler_template
23
31
  @raw_sampler_template ||= File.read(File.expand_path('../dump/netflow.v9.sampler_template.dump', __FILE__))
24
32
  end
@@ -27,6 +35,10 @@ class Netflow9ParserTest < Test::Unit::TestCase
27
35
  @raw_sampler_data ||= File.read(File.expand_path('../dump/netflow.v9.sampler.dump', __FILE__))
28
36
  end
29
37
 
38
+ def raw_2byte_as_template
39
+ @raw_2byte_as_template ||= File.read(File.expand_path('../dump/netflow.v9.template.as2.dump', __FILE__))
40
+ end
41
+
30
42
  DEFAULT_HOST = '127.0.0.1'
31
43
 
32
44
  test 'parse netflow v9 binary data before loading corresponding template' do
@@ -127,4 +139,32 @@ class Netflow9ParserTest < Test::Unit::TestCase
127
139
  assert_equal nil, parsed.first[1]['sampling_algorithm']
128
140
  assert_equal nil, parsed.first[1]['sampling_interval']
129
141
  end
142
+
143
+ test 'parse netflow v9 binary data with templates whose AS field length varies' do
144
+ parser = create_parser
145
+
146
+ parsed = []
147
+ [raw_2byte_as_template, raw_template].each {|raw| parser.call(raw, DEFAULT_HOST){} }
148
+ parser.call(raw_data, DEFAULT_HOST) do |time, record|
149
+ parsed << [time, record]
150
+ end
151
+
152
+ assert_equal 1, parsed.size
153
+ assert_equal 0, parsed.first[1]['src_as']
154
+ assert_equal 65000, parsed.first[1]['dst_as']
155
+ end
156
+
157
+ test 'parse netflow v9 binary data contains mpls information' do
158
+ parser = create_parser
159
+
160
+ parsed = []
161
+ [raw_sampler_template, raw_sampler_data, raw_mpls_template].each {|raw| parser.call(raw, DEFAULT_HOST){} }
162
+ parser.call(raw_mpls_data, DEFAULT_HOST) do |time, record|
163
+ parsed << [time, record]
164
+ end
165
+
166
+ assert_equal 24002, parsed.first[1]['mpls_label_1']
167
+ assert_equal '192.168.32.100', parsed.first[1]['ipv4_src_addr']
168
+ assert_equal '172.16.32.2', parsed.first[1]['ipv4_dst_addr']
169
+ end
130
170
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-netflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.2.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Masahiro Nakagawa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-24 00:00:00.000000000 Z
11
+ date: 2016-04-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -93,8 +93,11 @@ files:
93
93
  - lib/fluent/plugin/vash.rb
94
94
  - test/dump/netflow.v5.dump
95
95
  - test/dump/netflow.v9.dump
96
+ - test/dump/netflow.v9.mpls-data.dump
97
+ - test/dump/netflow.v9.mpls-template.dump
96
98
  - test/dump/netflow.v9.sampler.dump
97
99
  - test/dump/netflow.v9.sampler_template.dump
100
+ - test/dump/netflow.v9.template.as2.dump
98
101
  - test/dump/netflow.v9.template.dump
99
102
  - test/helper.rb
100
103
  - test/test_in_netflow.rb
@@ -127,8 +130,11 @@ summary: Netflow plugin for Fluentd
127
130
  test_files:
128
131
  - test/dump/netflow.v5.dump
129
132
  - test/dump/netflow.v9.dump
133
+ - test/dump/netflow.v9.mpls-data.dump
134
+ - test/dump/netflow.v9.mpls-template.dump
130
135
  - test/dump/netflow.v9.sampler.dump
131
136
  - test/dump/netflow.v9.sampler_template.dump
137
+ - test/dump/netflow.v9.template.as2.dump
132
138
  - test/dump/netflow.v9.template.dump
133
139
  - test/helper.rb
134
140
  - test/test_in_netflow.rb