logstash-codec-netflow 3.2.2 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 05a98ccdb2cc8a75bdda0d5186e17385e4fbb3f2
4
- data.tar.gz: e78c49964d3d50a6904895ec9329da7607dcfc81
3
+ metadata.gz: c13bc81edc56109e3a9c795d38d565d68b40ae44
4
+ data.tar.gz: 30620977b21b47ce7b77c2d54427b34e060ed5c9
5
5
  SHA512:
6
- metadata.gz: 84be91763bb7159eefb3f924176955deca396b95fbe09a189c1554d892bac06f34591b2147d91302964b29e69441753139e0f859bbc7741b99ec51ba38f0e642
7
- data.tar.gz: a2530632510c75b0aa30a153f1f3f199a9c178777e376d2a9dcaf602140de9d866e94775b230e9955c6cbb0921e97aa6da9187d909830a85906b1f82b76b6531
6
+ metadata.gz: 88656d03bee9bcae65cfe6b0a25830cd5f2d0831c369d38e7e8f24b1dea976bf391d8197d3c355670ac012244a78c958ed6472d9c31514c00b7decf0fcad7559
7
+ data.tar.gz: c269177cae5252f6bb7fc7d475b36c9c956fa71b5a21468c1ad5d0a537b80f7c89ed7a6ea244191f0af77fa85e65ec801a9d4c1b03aaa32f6bbaac3ee49d3cb3
data/CHANGELOG.md CHANGED
@@ -1,3 +1,21 @@
1
+ ## 3.3.0
2
+
3
+ - Added support for Cisco ASR 9000 (Netflow v9)
4
+
5
+ ## 3.2.5
6
+
7
+ - Added support for Streamcore StreamGroomer (Netflow v9)
8
+ - Fixed docs so they can generate
9
+
10
+ ## 3.2.4
11
+
12
+ - Fixed 0-length template field length (Netflow 9)
13
+
14
+ ## 3.2.3
15
+
16
+ - Fixed 0-length scope field length (Netflow 9, Juniper SRX)
17
+ - Fixed JRuby 9K compatibility
18
+
1
19
  ## 3.2.2
2
20
 
3
21
  - Added support for VMware VDS IPFIX although field definitions are unknown
data/CONTRIBUTORS CHANGED
@@ -10,10 +10,12 @@ Contributors:
10
10
  * G.J. Moed (gjmoed)
11
11
  * Jordan Sissel (jordansissel)
12
12
  * Jorrit Folmer (jorritfolmer)
13
+ * Keenan Tims (ktims)
13
14
  * Matt Dainty (bodgit)
14
15
  * Paul Warren (pwarren)
15
16
  * Pier-Hugues Pellerin (ph)
16
17
  * Pulkit Agrawal (propulkit)
18
+ * Raju Nair (rajutech76)
17
19
  * Richard Pijnenburg (electrical)
18
20
  * Salvador Ferrer (salva-ferrer)
19
21
  * Will Rigby (wrigby)
@@ -30,8 +30,8 @@ require "logstash/json"
30
30
  #
31
31
  # Example Logstash configuration:
32
32
  #
33
- # [source]
34
- # -----------------------------
33
+ # [source, ruby]
34
+ # --------------------------
35
35
  # input {
36
36
  # udp {
37
37
  # host => localhost
@@ -43,24 +43,24 @@ require "logstash/json"
43
43
  # }
44
44
  # udp {
45
45
  # host => localhost
46
- # port => 4739
47
- # codec => netflow {
46
+ # port => 4739
47
+ # codec => netflow {
48
48
  # versions => [10]
49
49
  # target => ipfix
50
- # }
51
- # type => ipfix
50
+ # }
51
+ # type => ipfix
52
52
  # }
53
53
  # tcp {
54
54
  # host => localhost
55
55
  # port => 4739
56
56
  # codec => netflow {
57
57
  # versions => [10]
58
- # target => ipfix
58
+ # target => ipfix
59
59
  # }
60
60
  # type => ipfix
61
61
  # }
62
62
  # }
63
- # -----------------------------
63
+ # --------------------------
64
64
 
65
65
  class LogStash::Codecs::Netflow < LogStash::Codecs::Base
66
66
  config_name "netflow"
@@ -89,15 +89,17 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
89
89
  #
90
90
  # Each Netflow field is defined like so:
91
91
  #
92
- # ---
93
- # id:
94
- # - default length in bytes
95
- # - :name
96
- # id:
97
- # - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
98
- # - :name
99
- # id:
100
- # - :skip
92
+ # [source,yaml]
93
+ # --------------------------
94
+ # id:
95
+ # - default length in bytes
96
+ # - :name
97
+ # id:
98
+ # - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
99
+ # - :name
100
+ # id:
101
+ # - :skip
102
+ # --------------------------
101
103
  #
102
104
  # See <https://github.com/logstash-plugins/logstash-codec-netflow/blob/master/lib/logstash/codecs/netflow/netflow.yaml> for the base set.
103
105
  config :netflow_definitions, :validate => :path
@@ -107,13 +109,15 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
107
109
  # Very similar to the Netflow version except there is a top level Private
108
110
  # Enterprise Number (PEN) key added:
109
111
  #
110
- # ---
111
- # pen:
112
- # id:
113
- # - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
114
- # - :name
115
- # id:
116
- # - :skip
112
+ # [source,yaml]
113
+ # --------------------------
114
+ # pen:
115
+ # id:
116
+ # - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
117
+ # - :name
118
+ # id:
119
+ # - :skip
120
+ # --------------------------
117
121
  #
118
122
  # There is an implicit PEN 0 for the standard fields.
119
123
  #
@@ -186,6 +190,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
186
190
  yield(decode_netflow5(flowset, record))
187
191
  end
188
192
  elsif header.version == 9
193
+ # BinData::trace_reading do
189
194
  flowset = Netflow9PDU.read(payload)
190
195
  flowset.records.each do |record|
191
196
  if metadata != nil
@@ -193,7 +198,8 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
193
198
  else
194
199
  decode_netflow9(flowset, record).each{|event| yield(event)}
195
200
  end
196
- end
201
+ # end
202
+ end
197
203
  elsif header.version == 10
198
204
  flowset = IpfixPDU.read(payload)
199
205
  flowset.records.each do |record|
@@ -254,21 +260,28 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
254
260
  record.flowset_data.templates.each do |template|
255
261
  catch (:field) do
256
262
  fields = []
263
+ template_length = 0
257
264
  # Template flowset (0) or Options template flowset (1) ?
258
265
  if record.flowset_id == 0
259
266
  template.record_fields.each do |field|
260
- entry = netflow_field_for(field.field_type, field.field_length)
261
- throw :field unless entry
262
- fields += entry
267
+ if field.field_length > 0
268
+ entry = netflow_field_for(field.field_type, field.field_length, template.template_id)
269
+ throw :field unless entry
270
+ fields += entry
271
+ template_length += field.field_length
272
+ end
263
273
  end
264
274
  else
265
275
  template.scope_fields.each do |field|
266
- fields << [uint_field(0, field.field_length), NETFLOW9_SCOPES[field.field_type]]
276
+ if field.field_length > 0
277
+ fields << [uint_field(0, field.field_length), NETFLOW9_SCOPES[field.field_type]]
278
+ end
267
279
  end
268
280
  template.option_fields.each do |field|
269
- entry = netflow_field_for(field.field_type, field.field_length)
281
+ entry = netflow_field_for(field.field_type, field.field_length, template.template_id)
270
282
  throw :field unless entry
271
283
  fields += entry
284
+ template_length += field.field_length
272
285
  end
273
286
  end
274
287
  # We get this far, we have a list of fields
@@ -279,6 +292,11 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
279
292
  key = "#{flowset.source_id}|#{template.template_id}"
280
293
  end
281
294
  @netflow_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
295
+ @logger.debug("Received template #{template.template_id} with fields #{fields.inspect}")
296
+ @logger.debug("Received template #{template.template_id} of size #{template_length} bytes. Representing in #{@netflow_templates[key].num_bytes} BinData bytes")
297
+ if template_length != @netflow_templates[key].num_bytes
298
+ @logger.warn("Received template #{template.template_id} of size (#{template_length} bytes) doesn't match BinData representation we built (#{@netflow_templates[key].num_bytes} bytes)")
299
+ end
282
300
  # Purge any expired templates
283
301
  @netflow_templates.cleanup!
284
302
  if @cache_save_path
@@ -300,7 +318,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
300
318
  unless template
301
319
  #@logger.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
302
320
  @logger.warn("No matching template for flow id #{record.flowset_id}")
303
- next
321
+ return events
304
322
  end
305
323
 
306
324
  length = record.flowset_length - 4
@@ -309,7 +327,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
309
327
  # be at most 3 padding bytes
310
328
  if template.num_bytes > length or ! (length % template.num_bytes).between?(0, 3)
311
329
  @logger.warn("Template length doesn't fit cleanly into flowset", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length)
312
- next
330
+ return events
313
331
  end
314
332
 
315
333
  array = BinData::Array.new(:type => template, :initial_length => length / template.num_bytes)
@@ -389,7 +407,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
389
407
 
390
408
  unless template
391
409
  @logger.warn("No matching template for flow id #{record.flowset_id}")
392
- next
410
+ return events
393
411
  end
394
412
 
395
413
  array = BinData::Array.new(:type => template, :read_until => :eof)
@@ -507,23 +525,34 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
507
525
  field
508
526
  end # def string_field
509
527
 
510
- def netflow_field_for(type, length)
528
+ def netflow_field_for(type, length, template_id)
511
529
  if @netflow_fields.include?(type)
512
530
  field = @netflow_fields[type].clone
513
531
  if field.is_a?(Array)
514
532
 
515
533
  field[0] = uint_field(length, field[0]) if field[0].is_a?(Integer)
516
534
 
517
- # Small bit of fixup for skip or string field types where the length
518
- # is dynamic
519
- case field[0]
535
+ # Small bit of fixup for:
536
+ # - skip or string field types where the length is dynamic
537
+ # - for uint(8|16|24|32} where we use the length as specified by the
538
+ # template instead of the YAML (e.g. ipv6_flow_label is 3 bytes in
539
+ # the YAML and Cisco doc, but Cisco ASR9k sends 4 bytes)
540
+ case field[0]
541
+ when :uint8
542
+ field[0] = uint_field(length, field[0])
543
+ when :uint16
544
+ field[0] = uint_field(length, field[0])
545
+ when :uint24
546
+ field[0] = uint_field(length, field[0])
547
+ when :uint32
548
+ field[0] = uint_field(length, field[0])
520
549
  when :skip
521
550
  field += [nil, {:length => length.to_i}]
522
551
  when :string
523
552
  field += [{:length => length.to_i, :trim_padding => true}]
524
553
  end
525
554
 
526
- @logger.debug? and @logger.debug("Definition complete", :field => field)
555
+ @logger.debug? and @logger.debug("Field definition complete for template #{template_id}", :field => field)
527
556
 
528
557
  [field]
529
558
  else
@@ -531,7 +560,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
531
560
  nil
532
561
  end
533
562
  else
534
- @logger.warn("Unsupported field", :type => type, :length => length)
563
+ @logger.warn("Unsupported field in template #{template_id}", :type => type, :length => length)
535
564
  nil
536
565
  end
537
566
  end # def netflow_field_for
@@ -189,8 +189,6 @@
189
189
  64:
190
190
  - :uint32
191
191
  - :ipv6_option_headers
192
- 64:
193
- - :skip
194
192
  65:
195
193
  - :skip
196
194
  66:
@@ -234,6 +232,9 @@
234
232
  148:
235
233
  - :uint32
236
234
  - :conn_id
235
+ 152:
236
+ - 8
237
+ - :flow_start_msec
237
238
  176:
238
239
  - :uint8
239
240
  - :icmp_type
@@ -246,6 +247,18 @@
246
247
  179:
247
248
  - :uint8
248
249
  - :icmp_code_ipv6
250
+ 180:
251
+ - :uint16
252
+ - :udp_src_port
253
+ 181:
254
+ - :uint16
255
+ - :udp_dst_port
256
+ 182:
257
+ - :uint16
258
+ - :tcp_src_port
259
+ 183:
260
+ - :uint16
261
+ - :tcp_dst_port
249
262
  201:
250
263
  - mpls_label_stack_octets
251
264
  - mpls_label_stack_octets
@@ -261,36 +274,132 @@
261
274
  228:
262
275
  - :uint16
263
276
  - :xlate_dst_port
277
+ 231:
278
+ - :uint32
279
+ - :fwd_flow_delta_bytes
280
+ 232:
281
+ - :uint32
282
+ - :rev_flow_delta_bytes
264
283
  233:
265
284
  - :uint8
266
285
  - :fw_event
286
+ 234:
287
+ - :uint32
288
+ - :ingressVRFID
289
+ 235:
290
+ - :uint32
291
+ - :egressVRFID
292
+ 236:
293
+ - :string
294
+ - :VRFname
267
295
  281:
268
296
  - :ip6_addr
269
297
  - :xlate_src_addr_ipv6
270
298
  282:
271
299
  - :ip6_addr
272
300
  - :xlate_dst_addr_ipv6
273
- 33002:
274
- - :uint16
275
- - :fw_ext_event
276
301
  323:
277
302
  - 8
278
303
  - :event_time_msec
279
- 152:
280
- - 8
281
- - :flow_start_msec
282
- 231:
283
- - :uint32
284
- - :fwd_flow_delta_bytes
285
- 232:
304
+ 361:
305
+ - :uint16
306
+ - :postNATPortBlockStart
307
+ 362:
308
+ - :uint16
309
+ - :postNATPortBlockEnd
310
+ 8192:
286
311
  - :uint32
287
- - :rev_flow_delta_bytes
312
+ - :streamcore_wan_rtt
313
+ 8193:
314
+ - :uint32
315
+ - :streamcore_net_app_resp_time
316
+ 8194:
317
+ - :uint32
318
+ - :streamcore_total_app_resp_time
319
+ 8195:
320
+ - :uint16
321
+ - :streamcore_tcp_retrans_rate
322
+ 8196:
323
+ - :uint8
324
+ - :streamcore_call_direction
325
+ 8256:
326
+ - :string
327
+ - :streamcore_hostname
328
+ 8257:
329
+ - :string
330
+ - :streamcore_url
331
+ 8258:
332
+ - :string
333
+ - :streamcore_ssl_cn
334
+ 8259:
335
+ - :string
336
+ - :streamcore_ssl_org
337
+ 8320:
338
+ - :uint16
339
+ - :streamcore_mos_lq
340
+ 8321:
341
+ - :uint16
342
+ - :streamcore_net_delay
343
+ 8322:
344
+ - :uint16
345
+ - :streamcore_net_loss
346
+ 8323:
347
+ - :uint16
348
+ - :streamcore_net_jitter
349
+ 8324:
350
+ - :uint16
351
+ - :streamcore_net_discard
352
+ 8325:
353
+ - :uint8
354
+ - :streamcore_rtp_clockrate_in
355
+ 8326:
356
+ - :uint8
357
+ - :streamcore_rtp_clockrate_out
358
+ 8327:
359
+ - :uint8
360
+ - :streamcore_codec_in
361
+ 8328:
362
+ - :uint8
363
+ - :streamcore_codec_out
364
+ 8384:
365
+ - :uint32
366
+ - :streamcore_id_rule_1
367
+ 8385:
368
+ - :uint32
369
+ - :streamcore_id_rule_2
370
+ 8386:
371
+ - :uint32
372
+ - :streamcore_id_rule_3
373
+ 8387:
374
+ - :uint32
375
+ - :streamcore_id_rule_4
376
+ 8388:
377
+ - :uint32
378
+ - :streamcore_id_rule_5
379
+ 8389:
380
+ - :uint32
381
+ - :streamcore_id_rule_6
382
+ 8390:
383
+ - :uint32
384
+ - :streamcore_id_rule_7
385
+ 8391:
386
+ - :uint32
387
+ - :streamcore_id_rule_8
388
+ 8392:
389
+ - :uint32
390
+ - :streamcore_id_rule_9
391
+ 8393:
392
+ - :uint32
393
+ - :streamcore_id_rule_10
288
394
  33000:
289
395
  - :acl_id_asa
290
396
  - :ingress_acl_id
291
397
  33001:
292
398
  - :acl_id_asa
293
399
  - egress_acl_id
400
+ 33002:
401
+ - :uint16
402
+ - :fw_ext_event
294
403
  40000:
295
404
  - :string
296
405
  - :username
@@ -309,18 +418,3 @@
309
418
  40005:
310
419
  - :uint8
311
420
  - :fw_event
312
- 95:
313
- - :uint32
314
- - :application_id
315
- 180:
316
- - :uint16
317
- - :udp_src_port
318
- 181:
319
- - :uint16
320
- - :udp_dst_port
321
- 182:
322
- - :uint16
323
- - :tcp_src_port
324
- 183:
325
- - :uint16
326
- - :tcp_dst_port
@@ -179,15 +179,15 @@ class NetflowOptionFlowset < BinData::Record
179
179
  endian :big
180
180
  array :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
181
181
  uint16 :template_id
182
- uint16 :scope_length
183
- uint16 :option_length
182
+ uint16 :scope_length, :assert => lambda { scope_length > 0 }
183
+ uint16 :option_length, :assert => lambda { option_length > 0 }
184
184
  array :scope_fields, :initial_length => lambda { scope_length / 4 } do
185
185
  uint16 :field_type
186
186
  uint16 :field_length
187
187
  end
188
188
  array :option_fields, :initial_length => lambda { option_length / 4 } do
189
189
  uint16 :field_type
190
- uint16 :field_length
190
+ uint16 :field_length, :assert => lambda { field_length > 0 }
191
191
  end
192
192
  end
193
193
  skip :length => lambda { templates.length.odd? ? 2 : 0 }
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-netflow'
4
- s.version = '3.2.2'
4
+ s.version = '3.3.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows."
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -966,6 +966,186 @@ describe LogStash::Codecs::Netflow do
966
966
 
967
967
  end
968
968
 
969
+ context "Netflow 9 Streamcore" do
970
+ let(:data) do
971
+ packets = []
972
+ packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_streamcore_tpl_data256.dat"), :mode => "rb")
973
+ packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_streamcore_tpl_data260.dat"), :mode => "rb")
974
+ end
975
+
976
+ let(:json_events) do
977
+ events = []
978
+ events << <<-END
979
+ {
980
+ "netflow": {
981
+ "in_pkts": 3,
982
+ "first_switched": "2017-01-11T11:47:23.999Z",
983
+ "flowset_id": 256,
984
+ "l4_src_port": 8080,
985
+ "streamcore_id_rule_1": 1171,
986
+ "streamcore_id_rule_2": 1179,
987
+ "in_bytes": 128,
988
+ "protocol": 6,
989
+ "streamcore_id_rule_5": 0,
990
+ "tcp_flags": 19,
991
+ "streamcore_id_rule_3": 1192,
992
+ "streamcore_id_rule_4": 1435,
993
+ "streamcore_net_app_resp_time": 0,
994
+ "l4_dst_port": 50073,
995
+ "output_snmp": 1148,
996
+ "streamcore_call_direction": 1,
997
+ "src_tos": 40,
998
+ "ipv4_dst_addr": "10.231.128.150",
999
+ "version": 9,
1000
+ "streamcore_tcp_retrans_rate": 0,
1001
+ "flow_seq_num": 2143054578,
1002
+ "ipv4_src_addr": "100.78.40.201",
1003
+ "input_snmp": 1152,
1004
+ "last_switched": "2017-01-11T11:47:29.999Z",
1005
+ "streamcore_wan_rtt": 0,
1006
+ "streamcore_total_app_resp_time": 0
1007
+ },
1008
+ "@timestamp": "2017-01-11T11:48:15.000Z",
1009
+ "@version": "1"
1010
+ }
1011
+ END
1012
+
1013
+ events << <<-END
1014
+ {
1015
+ "netflow": {
1016
+ "in_pkts": 4,
1017
+ "first_switched": "2017-01-11T11:47:23.999Z",
1018
+ "flowset_id": 256,
1019
+ "l4_src_port": 50073,
1020
+ "streamcore_id_rule_1": 1171,
1021
+ "streamcore_id_rule_2": 1179,
1022
+ "in_bytes": 172,
1023
+ "protocol": 6,
1024
+ "streamcore_id_rule_5": 0,
1025
+ "tcp_flags": 19,
1026
+ "streamcore_id_rule_3": 1192,
1027
+ "streamcore_id_rule_4": 1435,
1028
+ "streamcore_net_app_resp_time": 0,
1029
+ "l4_dst_port": 8080,
1030
+ "output_snmp": 1152,
1031
+ "streamcore_call_direction": 0,
1032
+ "src_tos": 40,
1033
+ "ipv4_dst_addr": "100.78.40.201",
1034
+ "version": 9,
1035
+ "streamcore_tcp_retrans_rate": 0,
1036
+ "flow_seq_num": 2143054578,
1037
+ "ipv4_src_addr": "10.231.128.150",
1038
+ "input_snmp": 1148,
1039
+ "last_switched": "2017-01-11T11:47:29.999Z",
1040
+ "streamcore_wan_rtt": 0,
1041
+ "streamcore_total_app_resp_time": 0
1042
+ },
1043
+ "@timestamp": "2017-01-11T11:48:15.000Z",
1044
+ "@version": "1"
1045
+ }
1046
+ END
1047
+
1048
+ events << <<-END
1049
+ {
1050
+ "netflow": {
1051
+ "streamcore_id_rule_10": 0,
1052
+ "in_pkts": 10,
1053
+ "first_switched": "2017-01-11T11:22:44.999Z",
1054
+ "flowset_id": 260,
1055
+ "l4_src_port": 8080,
1056
+ "reamcore_id_rule_1": 1171,
1057
+ "streamcore_id_rule_2": 1179,
1058
+ "in_bytes": 3943,
1059
+ "protocol": 6,
1060
+ "streamcore_id_rule_5": 0,
1061
+ "tcp_flags": 26,
1062
+ "streamcore_id_rule_6": 0,
1063
+ "streamcore_id_rule_3": 1192,
1064
+ "streamcore_id_rule_4": 1435,
1065
+ "streamcore_id_rule_9": 0,
1066
+ "streamcore_id_rule_7": 0,
1067
+ "streamcore_id_rule_8": 0,
1068
+ "streamcore_net_app_resp_time": 17,
1069
+ "l4_dst_port": 53483,
1070
+ "output_snmp": 1148,
1071
+ "streamcore_hostname": "live.lemde.fr",
1072
+ "streamcore_call_direction": 1,
1073
+ "src_tos": 40,
1074
+ "ipv4_dst_addr": "10.27.8.20",
1075
+ "version": 9,
1076
+ "streamcore_tcp_retrans_rate": 0,
1077
+ "flow_seq_num": 2142545188,
1078
+ "ipv4_src_addr": "100.78.40.201",
1079
+ "input_snmp": 1152,
1080
+ "last_switched": "2017-01-11T11:23:35.999Z",
1081
+ "streamcore_url": "\/mux.json",
1082
+ "streamcore_wan_rtt": 0,
1083
+ "streamcore_total_app_resp_time": 19
1084
+ },
1085
+ "@timestamp": "2017-01-11T11:23:51.000Z",
1086
+ "@version": "1"
1087
+ }
1088
+ END
1089
+
1090
+ events << <<-END
1091
+ {
1092
+ "netflow": {
1093
+ "streamcore_id_rule_10": 0,
1094
+ "in_pkts": 11,
1095
+ "first_switched": "2017-01-11T11:22:44.999Z",
1096
+ "flowset_id": 260,
1097
+ "l4_src_port": 53483,
1098
+ "streamcore_id_rule_1": 1171,
1099
+ "streamcore_id_rule_2": 1179,
1100
+ "in_bytes": 3052,
1101
+ "protocol": 6,
1102
+ "streamcore_id_rule_5": 0,
1103
+ "tcp_flags": 26,
1104
+ "streamcore_id_rule_6": 0,
1105
+ "streamcore_id_rule_3": 1192,
1106
+ "streamcore_id_rule_4": 1435,
1107
+ "streamcore_id_rule_9": 0,
1108
+ "streamcore_id_rule_7": 0,
1109
+ "streamcore_id_rule_8": 0,
1110
+ "streamcore_net_app_resp_time": 17,
1111
+ "l4_dst_port": 8080,
1112
+ "output_snmp": 1152,
1113
+ "streamcore_hostname": "live.lemde.fr",
1114
+ "streamcore_call_direction": 0,
1115
+ "src_tos": 40,
1116
+ "ipv4_dst_addr": "100.78.40.201",
1117
+ "version": 9,
1118
+ "streamcore_tcp_retrans_rate": 0,
1119
+ "flow_seq_num": 2142545188,
1120
+ "ipv4_src_addr": "10.27.8.20",
1121
+ "input_snmp": 1148,
1122
+ "last_switched": "2017-01-11T11:23:35.999Z",
1123
+ "streamcore_url": "\/mux.json",
1124
+ "streamcore_wan_rtt": 0,
1125
+ "streamcore_total_app_resp_time": 19
1126
+ },
1127
+ "@timestamp": "2017-01-11T11:23:51.000Z",
1128
+ "@version": "1"
1129
+ }
1130
+ END
1131
+
1132
+ events.map{|event| event.gsub(/\s+/, "")}
1133
+ end
1134
+
1135
+ it "should decode raw data" do
1136
+ expect(decode.size).to eq(4)
1137
+ expect(decode[0].get("[netflow][streamcore_id_rule_1]")).to eq(1171)
1138
+ expect(decode[3].get("[netflow][streamcore_hostname]")).to eq("live.lemde.fr")
1139
+ end
1140
+
1141
+ it "should serialize to json" do
1142
+ expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
1143
+ expect(JSON.parse(decode[3].to_json)).to eq(JSON.parse(json_events[3]))
1144
+ end
1145
+
1146
+ end
1147
+
1148
+
969
1149
  context "IPFIX Netscaler with variable length fields" do
970
1150
  let(:data) do
971
1151
  # this ipfix raw data was produced by a Netscaler appliance and captured with wireshark
@@ -1236,14 +1416,192 @@ describe LogStash::Codecs::Netflow do
1236
1416
 
1237
1417
  it "should serialize to json" do
1238
1418
  expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
1239
- expect(JSON.parse(decode[1].to_json)).to eq(JSON.parse(json_events[1]))
1240
- expect(JSON.parse(decode[2].to_json)).to eq(JSON.parse(json_events[2]))
1241
- expect(JSON.parse(decode[3].to_json)).to eq(JSON.parse(json_events[3]))
1242
- expect(JSON.parse(decode[4].to_json)).to eq(JSON.parse(json_events[4]))
1243
1419
  end
1244
1420
 
1245
1421
  end
1246
1422
 
1423
+ context "Juniper SRX options template with 0 scope field length" do
1424
+ let(:data) do
1425
+ packets = []
1426
+ packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_juniper_srx_tplopt.dat"), :mode => "rb")
1427
+ end
1428
+
1429
+ let(:json_events) do
1430
+ events = []
1431
+ events << <<-END
1432
+ {
1433
+ "netflow": {
1434
+ "flow_seq_num": 338,
1435
+ "flowset_id": 256,
1436
+ "version":9,
1437
+ "sampling_algorithm":2,
1438
+ "sampling_interval":1
1439
+ },
1440
+ "@timestamp":"2016-11-29T00:21:56.000Z",
1441
+ "@version":"1"
1442
+ }
1443
+ END
1444
+ events.map{|event| event.gsub(/\s+/, "")}
1445
+ end
1446
+
1447
+ it "should decode raw data" do
1448
+ expect(decode.size).to eq(1)
1449
+ expect(decode[0].get("[netflow][sampling_algorithm]")).to eq(2)
1450
+ end
1451
+
1452
+ it "should serialize to json" do
1453
+ expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
1454
+ end
1455
+
1456
+ end
1457
+
1458
+ context "Netflow 9 template with 0 length fields" do
1459
+ let(:data) do
1460
+ packets = []
1461
+ packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_0length_fields_tpl_data.dat"), :mode => "rb")
1462
+ end
1463
+
1464
+ let(:json_events) do
1465
+ events = []
1466
+ events << <<-END
1467
+ {
1468
+ "netflow":{
1469
+ "output_snmp":3,
1470
+ "dst_mask":32,
1471
+ "in_pkts":0,
1472
+ "ipv4_dst_addr":"239.255.255.250",
1473
+ "first_switched":"2016-12-23T01:34:52.999Z",
1474
+ "flowset_id":256,
1475
+ "l4_src_port":0,
1476
+ "src_mask":32,
1477
+ "version":9,
1478
+ "flow_seq_num":100728833,
1479
+ "ipv4_src_addr":"192.168.1.33",
1480
+ "in_bytes":0,
1481
+ "protocol":2,
1482
+ "input_snmp":2,
1483
+ "last_switched":"2016-12-23T01:34:52.999Z",
1484
+ "tcp_flags":0,
1485
+ "engine_id":1,
1486
+ "out_pkts":1,
1487
+ "out_bytes":32,
1488
+ "l4_dst_port":0,
1489
+ "direction":1
1490
+ },
1491
+ "@timestamp":"2016-12-23T01:35:31.000Z",
1492
+ "@version":"1"
1493
+ }
1494
+ END
1495
+ events.map{|event| event.gsub(/\s+/, "")}
1496
+ end
1497
+
1498
+ it "should decode raw data" do
1499
+ expect(decode.size).to eq(10)
1500
+ expect(decode[9].get("[netflow][ipv4_src_addr]")).to eq("192.168.1.33")
1501
+ end
1502
+
1503
+ it "should serialize to json" do
1504
+ expect(JSON.parse(decode[9].to_json)).to eq(JSON.parse(json_events[0]))
1505
+ end
1506
+
1507
+ end
1508
+
1509
+ context "Netflow 9 Cisco ASR 9000 series options template 256" do
1510
+ let(:data) do
1511
+ packets = []
1512
+ packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_cisco_asr9k_opttpl256.dat"), :mode => "rb")
1513
+ packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_cisco_asr9k_data256.dat"), :mode => "rb")
1514
+ end
1515
+
1516
+ let(:json_events) do
1517
+ events = []
1518
+ events << <<-END
1519
+ {
1520
+ "netflow": {
1521
+ "flow_seq_num": 24496783,
1522
+ "scope_system": 3250896451,
1523
+ "input_snmp": 104,
1524
+ "if_desc": "TenGigE0_6_0_2",
1525
+ "flowset_id": 256,
1526
+ "version": 9
1527
+ },
1528
+ "@timestamp": "2016-12-06T10:09:48.000Z",
1529
+ "@version": "1"
1530
+ }
1531
+ END
1532
+ events.map{|event| event.gsub(/\s+/, "")}
1533
+ end
1534
+
1535
+ it "should decode raw data" do
1536
+ expect(decode.size).to eq(19)
1537
+ expect(decode[18].get("[netflow][if_desc]")).to eq("TenGigE0_6_0_2")
1538
+ end
1539
+
1540
+ it "should serialize to json" do
1541
+ expect(JSON.parse(decode[18].to_json)).to eq(JSON.parse(json_events[0]))
1542
+ end
1543
+
1544
+ end
1545
+
1546
+ context "Netflow 9 Cisco ASR 9000 series template 260" do
1547
+ let(:data) do
1548
+ packets = []
1549
+ packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_cisco_asr9k_tpl260.dat"), :mode => "rb")
1550
+ packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_cisco_asr9k_data260.dat"), :mode => "rb")
1551
+ end
1552
+
1553
+ let(:json_events) do
1554
+ events = []
1555
+ events << <<-END
1556
+ {
1557
+ "netflow": {
1558
+ "dst_as": 64498,
1559
+ "forwarding_status": {
1560
+ "reason": 0,
1561
+ "status": 1
1562
+ },
1563
+ "in_pkts": 2,
1564
+ "first_switched": "2016-12-06T10:08:53.999Z",
1565
+ "flowset_id": 260,
1566
+ "l4_src_port": 443,
1567
+ "in_bytes": 112,
1568
+ "protocol": 6,
1569
+ "tcp_flags": 18,
1570
+ "ingressVRFID": 1610612736,
1571
+ "l4_dst_port": 52364,
1572
+ "src_as": 15169,
1573
+ "direction": 1,
1574
+ "output_snmp": 158,
1575
+ "dst_mask": 24,
1576
+ "ipv4_dst_addr": "10.0.15.38",
1577
+ "src_tos": 0,
1578
+ "src_mask": 24,
1579
+ "version": 9,
1580
+ "flow_seq_num": 24495777,
1581
+ "ipv4_src_addr": "10.0.29.46",
1582
+ "egressVRFID": 1610612736,
1583
+ "input_snmp": 75,
1584
+ "last_switched": "2016-12-06T10:08:54.999Z",
1585
+ "flow_sampler_id": 1,
1586
+ "bgp_ipv4_next_hop": "10.0.14.27"
1587
+ },
1588
+ "@timestamp": "2016-12-06T10:09:24.000Z",
1589
+ "@version": "1"
1590
+ }
1591
+ END
1592
+ events.map{|event| event.gsub(/\s+/, "")}
1593
+ end
1594
+
1595
+ it "should decode raw data" do
1596
+ expect(decode.size).to eq(21)
1597
+ expect(decode[20].get("[netflow][egressVRFID]")).to eq(1610612736)
1598
+ end
1599
+
1600
+ it "should serialize to json" do
1601
+ expect(JSON.parse(decode[20].to_json)).to eq(JSON.parse(json_events[0]))
1602
+ end
1603
+ end
1604
+
1247
1605
  end
1248
1606
 
1249
1607
  describe LogStash::Codecs::Netflow, 'missing templates, no template caching configured' do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-netflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.2.2
4
+ version: 3.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-12-25 00:00:00.000000000 Z
11
+ date: 2017-02-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -90,17 +90,28 @@ files:
90
90
  - spec/codecs/netflow5_test_invalid02.dat
91
91
  - spec/codecs/netflow5_test_juniper_mx80.dat
92
92
  - spec/codecs/netflow5_test_microtik.dat
93
+ - spec/codecs/netflow9_test_0length_fields_tpl_data.dat
93
94
  - spec/codecs/netflow9_test_cisco_asa_1_data.dat
94
95
  - spec/codecs/netflow9_test_cisco_asa_1_tpl.dat
95
96
  - spec/codecs/netflow9_test_cisco_asa_2_data.dat
96
97
  - spec/codecs/netflow9_test_cisco_asa_2_tpl_26x.dat
97
98
  - spec/codecs/netflow9_test_cisco_asa_2_tpl_27x.dat
99
+ - spec/codecs/netflow9_test_cisco_asr9k_data256.dat
100
+ - spec/codecs/netflow9_test_cisco_asr9k_data260.dat
101
+ - spec/codecs/netflow9_test_cisco_asr9k_opttpl256.dat
102
+ - spec/codecs/netflow9_test_cisco_asr9k_opttpl257.dat
103
+ - spec/codecs/netflow9_test_cisco_asr9k_opttpl334.dat
104
+ - spec/codecs/netflow9_test_cisco_asr9k_tpl260.dat
105
+ - spec/codecs/netflow9_test_cisco_asr9k_tpl266.dat
98
106
  - spec/codecs/netflow9_test_invalid01.dat
107
+ - spec/codecs/netflow9_test_juniper_srx_tplopt.dat
99
108
  - spec/codecs/netflow9_test_macaddr_data.dat
100
109
  - spec/codecs/netflow9_test_macaddr_tpl.dat
101
110
  - spec/codecs/netflow9_test_nprobe_data.dat
102
111
  - spec/codecs/netflow9_test_nprobe_tpl.dat
103
112
  - spec/codecs/netflow9_test_softflowd_tpl_data.dat
113
+ - spec/codecs/netflow9_test_streamcore_tpl_data256.dat
114
+ - spec/codecs/netflow9_test_streamcore_tpl_data260.dat
104
115
  - spec/codecs/netflow9_test_ubnt_edgerouter_data1024.dat
105
116
  - spec/codecs/netflow9_test_ubnt_edgerouter_data1025.dat
106
117
  - spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
@@ -147,17 +158,28 @@ test_files:
147
158
  - spec/codecs/netflow5_test_invalid02.dat
148
159
  - spec/codecs/netflow5_test_juniper_mx80.dat
149
160
  - spec/codecs/netflow5_test_microtik.dat
161
+ - spec/codecs/netflow9_test_0length_fields_tpl_data.dat
150
162
  - spec/codecs/netflow9_test_cisco_asa_1_data.dat
151
163
  - spec/codecs/netflow9_test_cisco_asa_1_tpl.dat
152
164
  - spec/codecs/netflow9_test_cisco_asa_2_data.dat
153
165
  - spec/codecs/netflow9_test_cisco_asa_2_tpl_26x.dat
154
166
  - spec/codecs/netflow9_test_cisco_asa_2_tpl_27x.dat
167
+ - spec/codecs/netflow9_test_cisco_asr9k_data256.dat
168
+ - spec/codecs/netflow9_test_cisco_asr9k_data260.dat
169
+ - spec/codecs/netflow9_test_cisco_asr9k_opttpl256.dat
170
+ - spec/codecs/netflow9_test_cisco_asr9k_opttpl257.dat
171
+ - spec/codecs/netflow9_test_cisco_asr9k_opttpl334.dat
172
+ - spec/codecs/netflow9_test_cisco_asr9k_tpl260.dat
173
+ - spec/codecs/netflow9_test_cisco_asr9k_tpl266.dat
155
174
  - spec/codecs/netflow9_test_invalid01.dat
175
+ - spec/codecs/netflow9_test_juniper_srx_tplopt.dat
156
176
  - spec/codecs/netflow9_test_macaddr_data.dat
157
177
  - spec/codecs/netflow9_test_macaddr_tpl.dat
158
178
  - spec/codecs/netflow9_test_nprobe_data.dat
159
179
  - spec/codecs/netflow9_test_nprobe_tpl.dat
160
180
  - spec/codecs/netflow9_test_softflowd_tpl_data.dat
181
+ - spec/codecs/netflow9_test_streamcore_tpl_data256.dat
182
+ - spec/codecs/netflow9_test_streamcore_tpl_data260.dat
161
183
  - spec/codecs/netflow9_test_ubnt_edgerouter_data1024.dat
162
184
  - spec/codecs/netflow9_test_ubnt_edgerouter_data1025.dat
163
185
  - spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat