logstash-codec-netflow 4.1.0 → 4.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/CONTRIBUTORS +1 -0
- data/docs/index.asciidoc +8 -6
- data/lib/logstash/codecs/netflow.rb +151 -80
- data/logstash-codec-netflow.gemspec +1 -1
- data/spec/codecs/benchmarks/flowStartMilliseconds.rb +73 -0
- data/spec/codecs/benchmarks/ipfix_bench_sonicwall.py +209 -0
- data/spec/codecs/benchmarks/ipfix_bench_yaf.py +1078 -0
- metadata +161 -153
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 05072723b2f9f9f78b2fb55268ddca4e189e641f
|
4
|
+
data.tar.gz: 0316e6c3a82fd2ac4355ef1cd7ede8a73b2b2e28
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 145c03e3407507b71affae3b337b0afe489e474816c33836d6d0df632ac7b840418257aa20d18df31621c2d9bd73b02bc7aa7d032191115bcf4a20e128754045
|
7
|
+
data.tar.gz: d8b4cb2f0305d9a5d0f07132c466125480a41518ba5925f29bf2401117abb27d58f51e7e49698da283db8c11c0667c4630eaece7fd01d1db37ce5d776b3ffb98
|
data/CHANGELOG.md
CHANGED
data/CONTRIBUTORS
CHANGED
data/docs/index.asciidoc
CHANGED
@@ -30,30 +30,32 @@ This codec supports:
|
|
30
30
|
* Netflow v9
|
31
31
|
* IPFIX
|
32
32
|
|
33
|
-
The following Netflow/IPFIX exporters
|
33
|
+
The following Netflow/IPFIX exporters have been seen and tested with the most recent version of the Netflow Codec:
|
34
34
|
|
35
35
|
[cols="6,^2,^2,^2,12",options="header"]
|
36
36
|
|===========================================================================================
|
37
37
|
|Netflow exporter | v5 | v9 | IPFIX | Remarks
|
38
|
-
|Barracuda Firewall | | | y |
|
38
|
+
|Barracuda Firewall | | | y | With support for Extended Uniflow
|
39
39
|
|Cisco ASA | | y | |
|
40
|
-
|Cisco ASR 1k | | |
|
40
|
+
|Cisco ASR 1k | | | N | Fails because of duplicate fields
|
41
41
|
|Cisco ASR 9k | | y | |
|
42
42
|
|Cisco IOS 12.x | | y | |
|
43
|
-
|Cisco ISR w/ HSL | |
|
43
|
+
|Cisco ISR w/ HSL | | N | | Fails because of duplicate fields, see: https://github.com/logstash-plugins/logstash-codec-netflow/issues/93
|
44
44
|
|Cisco WLC | | y | |
|
45
45
|
|Citrix Netscaler | | | y | Still some unknown fields, labeled netscalerUnknown<id>
|
46
46
|
|fprobe | y | | |
|
47
47
|
|Fortigate FortiOS | | y | |
|
48
48
|
|Huawei Netstream | | y | |
|
49
49
|
|ipt_NETFLOW | y | y | y |
|
50
|
-
|Juniper
|
50
|
+
|Juniper MX | y | | y | SW > 12.3R8. Fails to decode IPFIX from Junos 16.1 due to duplicate field names which we currently don't support.
|
51
51
|
|Mikrotik | y | | y | http://wiki.mikrotik.com/wiki/Manual:IP/Traffic_Flow
|
52
52
|
|nProbe | y | y | y | L7 DPI fields now also supported
|
53
53
|
|Nokia BRAS | | | y |
|
54
|
-
|OpenBSD pflow | y |
|
54
|
+
|OpenBSD pflow | y | N | y | http://man.openbsd.org/OpenBSD-current/man4/pflow.4
|
55
|
+
|Riverbed | | N | | Not supported due to field ID conflicts. Workaround available in the definitions directory over at Elastiflow <https://github.com/robcowart/elastiflow>
|
55
56
|
|Sandvine Procera PacketLogic| | | y | v15.1
|
56
57
|
|Softflowd | y | y | y | IPFIX supported in https://github.com/djmdjm/softflowd
|
58
|
+
|Sophos UTM | | | y |
|
57
59
|
|Streamcore Streamgroomer | | y | |
|
58
60
|
|Palo Alto PAN-OS | | y | |
|
59
61
|
|Ubiquiti Edgerouter X | | y | | With MPLS labels
|
@@ -50,11 +50,8 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
|
|
50
50
|
FLOWSET_ID = "flowset_id"
|
51
51
|
|
52
52
|
def initialize(params = {})
|
53
|
-
@file_cache_mutex = Mutex.new
|
54
53
|
super(params)
|
55
54
|
@threadsafe = true
|
56
|
-
@decode_mutex_netflow = Mutex.new
|
57
|
-
@decode_mutex_ipfix = Mutex.new
|
58
55
|
end
|
59
56
|
|
60
57
|
def clone
|
@@ -63,8 +60,9 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
|
|
63
60
|
|
64
61
|
def register
|
65
62
|
require "logstash/codecs/netflow/util"
|
66
|
-
|
67
|
-
@
|
63
|
+
|
64
|
+
@netflow_templates = TemplateRegistry.new(logger, @cache_ttl, @cache_save_path && "#{@cache_save_path}/netflow_templates.cache")
|
65
|
+
@ipfix_templates = TemplateRegistry.new(logger, @cache_ttl, @cache_save_path && "#{@cache_save_path}/ipfix_templates.cache")
|
68
66
|
|
69
67
|
# Path to default Netflow v9 field definitions
|
70
68
|
filename = ::File.expand_path('netflow/netflow.yaml', ::File.dirname(__FILE__))
|
@@ -73,32 +71,6 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
|
|
73
71
|
# Path to default IPFIX field definitions
|
74
72
|
filename = ::File.expand_path('netflow/ipfix.yaml', ::File.dirname(__FILE__))
|
75
73
|
@ipfix_fields = load_definitions(filename, @ipfix_definitions)
|
76
|
-
|
77
|
-
if @cache_save_path
|
78
|
-
if @versions.include?(9)
|
79
|
-
cache_save_file_netflow = "#{@cache_save_path}/netflow_templates.cache"
|
80
|
-
if File.exists?(cache_save_file_netflow)
|
81
|
-
raise "#{self.class.name}: Template cache file #{cache_save_file_netflow} not writable" unless File.writable?(cache_save_file_netflow)
|
82
|
-
@netflow_templates_cache = load_templates_cache("#{@cache_save_path}/netflow_templates.cache")
|
83
|
-
@netflow_templates_cache.each{ |key, fields| @netflow_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields) }
|
84
|
-
else
|
85
|
-
raise "#{self.class.name}: Template cache directory #{cache_save_path} not writable" unless File.writable?(cache_save_path)
|
86
|
-
@netflow_templates_cache = {}
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
if @versions.include?(10)
|
91
|
-
cache_save_file_ipfix = "#{@cache_save_path}/ipfix_templates.cache"
|
92
|
-
if File.exists?(cache_save_file_ipfix)
|
93
|
-
raise "#{self.class.name}: Template cache file #{cache_save_file_ipfix} not writable" unless File.writable?(cache_save_file_ipfix)
|
94
|
-
@ipfix_templates_cache = load_templates_cache("#{@cache_save_path}/ipfix_templates.cache")
|
95
|
-
@ipfix_templates_cache.each{ |key, fields| @ipfix_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields) }
|
96
|
-
else
|
97
|
-
raise "#{self.class.name}: Template cache directory #{cache_save_path} not writable" unless File.writable?(cache_save_path)
|
98
|
-
@ipfix_templates_cache = {}
|
99
|
-
end
|
100
|
-
end
|
101
|
-
end
|
102
74
|
end # def register
|
103
75
|
|
104
76
|
def decode(payload, metadata = nil, &block)
|
@@ -228,19 +200,11 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
|
|
228
200
|
else
|
229
201
|
key = "#{flowset.source_id}|#{template.template_id}"
|
230
202
|
end
|
231
|
-
|
232
|
-
@decode_mutex_netflow.synchronize do
|
233
|
-
@netflow_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
|
203
|
+
@netflow_templates.register(key, fields) do |bindata|
|
234
204
|
@logger.debug("Received template #{template.template_id} with fields #{fields.inspect}")
|
235
|
-
@logger.debug("Received template #{template.template_id} of size #{template_length} bytes. Representing in #{
|
236
|
-
if template_length !=
|
237
|
-
@logger.warn("Received template #{template.template_id} of size #{template_length} bytes doesn't match BinData representation we built (#{
|
238
|
-
end
|
239
|
-
# Purge any expired templates
|
240
|
-
@netflow_templates.cleanup!
|
241
|
-
if @cache_save_path
|
242
|
-
@netflow_templates_cache[key] = fields
|
243
|
-
save_templates_cache(@netflow_templates_cache, "#{@cache_save_path}/netflow_templates.cache")
|
205
|
+
@logger.debug("Received template #{template.template_id} of size #{template_length} bytes. Representing in #{bindata.num_bytes} BinData bytes")
|
206
|
+
if template_length != bindata.num_bytes
|
207
|
+
@logger.warn("Received template #{template.template_id} of size #{template_length} bytes doesn't match BinData representation we built (#{bindata.num_bytes} bytes)")
|
244
208
|
end
|
245
209
|
end
|
246
210
|
end
|
@@ -255,7 +219,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
|
|
255
219
|
key = "#{flowset.source_id}|#{record.flowset_id}"
|
256
220
|
end
|
257
221
|
|
258
|
-
template = @
|
222
|
+
template = @netflow_templates.fetch(key)
|
259
223
|
|
260
224
|
if !template
|
261
225
|
@logger.warn("Can't (yet) decode flowset id #{record.flowset_id} from source id #{flowset.source_id}, because no template to decode it with has been received. This message will usually go away after 1 minute.")
|
@@ -338,23 +302,14 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
|
|
338
302
|
end
|
339
303
|
# FIXME Source IP address required in key
|
340
304
|
key = "#{flowset.observation_domain_id}|#{template.template_id}"
|
341
|
-
|
342
|
-
@
|
343
|
-
@ipfix_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
|
344
|
-
# Purge any expired templates
|
345
|
-
@ipfix_templates.cleanup!
|
346
|
-
if @cache_save_path
|
347
|
-
@ipfix_templates_cache[key] = fields
|
348
|
-
save_templates_cache(@ipfix_templates_cache, "#{@cache_save_path}/ipfix_templates.cache")
|
349
|
-
end
|
350
|
-
end
|
305
|
+
|
306
|
+
@ipfix_templates.register(key, fields)
|
351
307
|
end
|
352
308
|
end
|
353
309
|
when 256..65535
|
354
310
|
# Data flowset
|
355
311
|
key = "#{flowset.observation_domain_id}|#{record.flowset_id}"
|
356
|
-
|
357
|
-
template = @decode_mutex_ipfix.synchronize { @ipfix_templates[key] }
|
312
|
+
template = @ipfix_templates.fetch(key)
|
358
313
|
|
359
314
|
if !template
|
360
315
|
@logger.warn("Can't (yet) decode flowset id #{record.flowset_id} from observation domain id #{flowset.observation_domain_id}, because no template to decode it with has been received. This message will usually go away after 1 minute.")
|
@@ -431,30 +386,6 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
|
|
431
386
|
fields
|
432
387
|
end
|
433
388
|
|
434
|
-
def load_templates_cache(file_path)
|
435
|
-
templates_cache = {}
|
436
|
-
begin
|
437
|
-
@logger.debug? and @logger.debug("Loading templates from template cache #{file_path}")
|
438
|
-
file_data = @file_cache_mutex.synchronize { File.read(file_path)}
|
439
|
-
templates_cache = JSON.parse(file_data)
|
440
|
-
rescue Exception => e
|
441
|
-
raise "#{self.class.name}: templates cache file could not be read @ (#{file_path}: #{e.class.name} #{e.message})"
|
442
|
-
end
|
443
|
-
|
444
|
-
templates_cache
|
445
|
-
end
|
446
|
-
|
447
|
-
def save_templates_cache(templates_cache, file_path)
|
448
|
-
begin
|
449
|
-
@logger.debug? and @logger.debug("Writing templates to template cache #{file_path}")
|
450
|
-
@file_cache_mutex.synchronize do
|
451
|
-
File.open(file_path, 'w') {|file| file.write templates_cache.to_json }
|
452
|
-
end
|
453
|
-
rescue Exception => e
|
454
|
-
raise "#{self.class.name}: saving templates cache file failed (#{file_path}) with error #{e}"
|
455
|
-
end
|
456
|
-
end
|
457
|
-
|
458
389
|
def uint_field(length, default)
|
459
390
|
# If length is 4, return :uint32, etc. and use default if length is 0
|
460
391
|
("uint" + (((length > 0) ? length : default) * 8).to_s).to_sym
|
@@ -599,4 +530,144 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
|
|
599
530
|
@logger.warn("Definition should be an array", :field => field)
|
600
531
|
end
|
601
532
|
end
|
533
|
+
|
534
|
+
class TemplateRegistry
|
535
|
+
##
|
536
|
+
# @param logger [Logger]
|
537
|
+
# @param ttl [Integer]
|
538
|
+
# @param file_path [String] (optional)
|
539
|
+
def initialize(logger, ttl, file_path=nil)
|
540
|
+
@logger = logger
|
541
|
+
@ttl = Integer(ttl)
|
542
|
+
@file_path = file_path
|
543
|
+
|
544
|
+
@mutex = Mutex.new
|
545
|
+
|
546
|
+
@bindata_struct_cache = Vash.new
|
547
|
+
@bindata_spec_cache = Vash.new
|
548
|
+
|
549
|
+
do_load unless file_path.nil?
|
550
|
+
end
|
551
|
+
|
552
|
+
##
|
553
|
+
# Register a Template by name using an array of type/name tuples.
|
554
|
+
#
|
555
|
+
# @param key [String]: the key under which to save this template
|
556
|
+
# @param field_tuples [Array<Array<String>>]: an array of [type,name] tuples, e.g., ["uint32","fieldName"]
|
557
|
+
# @return [BinData::Struct]
|
558
|
+
#
|
559
|
+
# If a block is given, the template is yielded to the block _before_ being saved in the cache.
|
560
|
+
#
|
561
|
+
# @yieldparam [BinData::Struct]
|
562
|
+
# @yieldreturn [void]
|
563
|
+
# @yieldthrow :invalid_template : if the template is deemed invalid within the block, throwing this symbol causes
|
564
|
+
# the template to not be cached.
|
565
|
+
#
|
566
|
+
# @threadsafe
|
567
|
+
def register(key, field_tuples, &block)
|
568
|
+
@mutex.synchronize do
|
569
|
+
do_register(key, field_tuples, &block)
|
570
|
+
end
|
571
|
+
end
|
572
|
+
|
573
|
+
##
|
574
|
+
# Fetch a Template by name
|
575
|
+
#
|
576
|
+
# @param key [String]
|
577
|
+
# @return [BinData::Struct]
|
578
|
+
#
|
579
|
+
# @threadsafe
|
580
|
+
def fetch(key)
|
581
|
+
@mutex.synchronize do
|
582
|
+
do_fetch(key)
|
583
|
+
end
|
584
|
+
end
|
585
|
+
|
586
|
+
##
|
587
|
+
# Force persist, potentially cleaning up elements from the file-based cache that have already been evicted from
|
588
|
+
# the memory-based cache
|
589
|
+
def persist()
|
590
|
+
@mutex.synchronize do
|
591
|
+
do_persist
|
592
|
+
end
|
593
|
+
end
|
594
|
+
|
595
|
+
private
|
596
|
+
attr_reader :logger
|
597
|
+
attr_reader :file_path
|
598
|
+
|
599
|
+
##
|
600
|
+
# @see `TemplateRegistry#register(String,Array<>)`
|
601
|
+
# @api private
|
602
|
+
def do_register(key, field_tuples)
|
603
|
+
template = BinData::Struct.new(:fields => field_tuples, :endian => :big)
|
604
|
+
|
605
|
+
catch(:invalid_template) do
|
606
|
+
yield(template) if block_given?
|
607
|
+
|
608
|
+
@bindata_spec_cache[key] = field_tuples
|
609
|
+
@bindata_struct_cache[key] = template
|
610
|
+
|
611
|
+
do_persist
|
612
|
+
|
613
|
+
template
|
614
|
+
end
|
615
|
+
end
|
616
|
+
|
617
|
+
##
|
618
|
+
# @api private
|
619
|
+
def do_load
|
620
|
+
unless File.exists?(file_path)
|
621
|
+
logger.warn('Template Cache does not exist', :file_path => file_path)
|
622
|
+
return
|
623
|
+
end
|
624
|
+
|
625
|
+
logger.debug? and logger.debug('Loading templates from template cache', :file_path => file_path)
|
626
|
+
file_data = File.read(file_path)
|
627
|
+
templates_cache = JSON.parse(file_data)
|
628
|
+
templates_cache.each do |key, fields|
|
629
|
+
do_register(key, fields)
|
630
|
+
end
|
631
|
+
|
632
|
+
logger.warn('Template Cache not writable', file_path: file_path) unless File.writable?(file_path)
|
633
|
+
rescue => e
|
634
|
+
logger.error('Template Cache could not be loaded', :file_path => file_path, :exception => e.message)
|
635
|
+
end
|
636
|
+
|
637
|
+
##
|
638
|
+
# @see `TemplateRegistry#persist`
|
639
|
+
# @api private
|
640
|
+
def do_persist
|
641
|
+
return if file_path.nil?
|
642
|
+
|
643
|
+
logger.debug? and logger.debug('Writing templates to template cache', :file_path => file_path)
|
644
|
+
|
645
|
+
fail('Template Cache not writable') if File.exists?(file_path) && !File.writable?(file_path)
|
646
|
+
|
647
|
+
do_cleanup!
|
648
|
+
|
649
|
+
templates_cache = @bindata_spec_cache
|
650
|
+
|
651
|
+
File.open(file_path, 'w') do |file|
|
652
|
+
file.write(templates_cache.to_json)
|
653
|
+
end
|
654
|
+
rescue Exception => e
|
655
|
+
logger.error('Template Cache could not be saved', :file_path => file_path, :exception => e.message)
|
656
|
+
end
|
657
|
+
|
658
|
+
##
|
659
|
+
# @see `TemplateRegistry#cleanup`
|
660
|
+
# @api private
|
661
|
+
def do_cleanup!
|
662
|
+
@bindata_spec_cache.cleanup!
|
663
|
+
@bindata_struct_cache.cleanup!
|
664
|
+
end
|
665
|
+
|
666
|
+
##
|
667
|
+
# @see `TemplateRegistry#fetch(String)`
|
668
|
+
# @api private
|
669
|
+
def do_fetch(key)
|
670
|
+
@bindata_struct_cache[key]
|
671
|
+
end
|
672
|
+
end
|
602
673
|
end # class LogStash::Filters::Netflow
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-netflow'
|
4
|
-
s.version = '4.1.
|
4
|
+
s.version = '4.1.1'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads Netflow v5, Netflow v9 and IPFIX data"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -0,0 +1,73 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'benchmark'
|
3
|
+
require 'bindata'
|
4
|
+
|
5
|
+
|
6
|
+
Benchmark.bm do |x|
|
7
|
+
x.report {
|
8
|
+
# Original IPFIX version, simplified
|
9
|
+
k = 'flowStartMilliseconds'
|
10
|
+
data = '1000'
|
11
|
+
v = BinData::String.new(:read_length => 4)
|
12
|
+
v.read(data)
|
13
|
+
2000000.times do
|
14
|
+
case k.to_s
|
15
|
+
when /^flow(?:Start|End)Seconds$/
|
16
|
+
event = 'blah'
|
17
|
+
when /^flow(?:Start|End)(Milli|Micro|Nano)seconds$/
|
18
|
+
case $1
|
19
|
+
when 'Milli'
|
20
|
+
event = v.snapshot.to_f / 1_000
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end }
|
24
|
+
|
25
|
+
x.report {
|
26
|
+
# Verion that omits v.snapshot, simplified
|
27
|
+
k = 'flowStartMilliseconds'
|
28
|
+
data = '1000'
|
29
|
+
v = BinData::String.new(:read_length => 4)
|
30
|
+
v.read(data)
|
31
|
+
2000000.times do
|
32
|
+
case k.to_s
|
33
|
+
when /^flow(?:Start|End)Seconds$/
|
34
|
+
event = 'blah'
|
35
|
+
when /^flow(?:Start|End)(Milli|Micro|Nano)seconds$/
|
36
|
+
case $1
|
37
|
+
when 'Milli'
|
38
|
+
event = data.to_f / 1_000
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end }
|
42
|
+
|
43
|
+
x.report {
|
44
|
+
# Original Netflow9 version, simplified
|
45
|
+
class MockFlowset < BinData::Record
|
46
|
+
endian :little
|
47
|
+
uint8 :uptime
|
48
|
+
uint8 :unix_sec
|
49
|
+
end
|
50
|
+
SWITCHED = /_switched$/
|
51
|
+
data1 = 'AB'
|
52
|
+
flowset = MockFlowset.read(data1)
|
53
|
+
k = 'first_switched'
|
54
|
+
v = 20
|
55
|
+
2000000.times do
|
56
|
+
case k.to_s
|
57
|
+
when SWITCHED
|
58
|
+
millis = flowset.uptime - v
|
59
|
+
seconds = flowset.unix_sec - (millis / 1000)
|
60
|
+
# v9 did away with the nanosecs field
|
61
|
+
micros = 1000000 - (millis % 1000)
|
62
|
+
event = v
|
63
|
+
else
|
64
|
+
event = 'blah'
|
65
|
+
end
|
66
|
+
end }
|
67
|
+
|
68
|
+
end
|
69
|
+
|
70
|
+
# user system total real
|
71
|
+
# 4.730000 0.000000 4.730000 ( 4.731333)
|
72
|
+
# 2.400000 0.000000 2.400000 ( 2.401072)
|
73
|
+
# 2.750000 0.000000 2.750000 ( 2.747525)
|
@@ -0,0 +1,209 @@
|
|
1
|
+
#!/usr/bin/env python2
|
2
|
+
import socket
|
3
|
+
import sys
|
4
|
+
import time
|
5
|
+
|
6
|
+
|
7
|
+
# IPFIX template
|
8
|
+
tpl = "000a00585b6b5242000010bda07e8c0000020048010000100001000400020004000400010008000400070002000a0004000b0002000c0004000e0004000f0004001500040016000400e1000400e2000400e3000200e40002".decode("hex")
|
9
|
+
'''
|
10
|
+
Cisco NetFlow/IPFIX
|
11
|
+
Version: 10
|
12
|
+
Length: 88
|
13
|
+
Timestamp: Aug 8, 2018 14:27:46.000000000 MDT
|
14
|
+
ExportTime: 1533760066
|
15
|
+
FlowSequence: 4285
|
16
|
+
Observation Domain Id: 2692647936
|
17
|
+
Set 1 [id=2] (Data Template): 256
|
18
|
+
FlowSet Id: Data Template (V10 [IPFIX]) (2)
|
19
|
+
FlowSet Length: 72
|
20
|
+
Template (Id = 256, Count = 16)
|
21
|
+
Template Id: 256
|
22
|
+
Field Count: 16
|
23
|
+
Field (1/16): BYTES
|
24
|
+
0... .... .... .... = Pen provided: No
|
25
|
+
.000 0000 0000 0001 = Type: BYTES (1)
|
26
|
+
Length: 4
|
27
|
+
Field (2/16): PKTS
|
28
|
+
0... .... .... .... = Pen provided: No
|
29
|
+
.000 0000 0000 0010 = Type: PKTS (2)
|
30
|
+
Length: 4
|
31
|
+
Field (3/16): PROTOCOL
|
32
|
+
0... .... .... .... = Pen provided: No
|
33
|
+
.000 0000 0000 0100 = Type: PROTOCOL (4)
|
34
|
+
Length: 1
|
35
|
+
Field (4/16): IP_SRC_ADDR
|
36
|
+
0... .... .... .... = Pen provided: No
|
37
|
+
.000 0000 0000 1000 = Type: IP_SRC_ADDR (8)
|
38
|
+
Length: 4
|
39
|
+
Field (5/16): L4_SRC_PORT
|
40
|
+
0... .... .... .... = Pen provided: No
|
41
|
+
.000 0000 0000 0111 = Type: L4_SRC_PORT (7)
|
42
|
+
Length: 2
|
43
|
+
Field (6/16): INPUT_SNMP
|
44
|
+
0... .... .... .... = Pen provided: No
|
45
|
+
.000 0000 0000 1010 = Type: INPUT_SNMP (10)
|
46
|
+
Length: 4
|
47
|
+
Field (7/16): L4_DST_PORT
|
48
|
+
0... .... .... .... = Pen provided: No
|
49
|
+
.000 0000 0000 1011 = Type: L4_DST_PORT (11)
|
50
|
+
Length: 2
|
51
|
+
Field (8/16): IP_DST_ADDR
|
52
|
+
0... .... .... .... = Pen provided: No
|
53
|
+
.000 0000 0000 1100 = Type: IP_DST_ADDR (12)
|
54
|
+
Length: 4
|
55
|
+
Field (9/16): OUTPUT_SNMP
|
56
|
+
0... .... .... .... = Pen provided: No
|
57
|
+
.000 0000 0000 1110 = Type: OUTPUT_SNMP (14)
|
58
|
+
Length: 4
|
59
|
+
Field (10/16): IP_NEXT_HOP
|
60
|
+
0... .... .... .... = Pen provided: No
|
61
|
+
.000 0000 0000 1111 = Type: IP_NEXT_HOP (15)
|
62
|
+
Length: 4
|
63
|
+
Field (11/16): LAST_SWITCHED
|
64
|
+
0... .... .... .... = Pen provided: No
|
65
|
+
.000 0000 0001 0101 = Type: LAST_SWITCHED (21)
|
66
|
+
Length: 4
|
67
|
+
Field (12/16): FIRST_SWITCHED
|
68
|
+
0... .... .... .... = Pen provided: No
|
69
|
+
.000 0000 0001 0110 = Type: FIRST_SWITCHED (22)
|
70
|
+
Length: 4
|
71
|
+
Field (13/16): postNATSourceIPv4Address
|
72
|
+
0... .... .... .... = Pen provided: No
|
73
|
+
.000 0000 1110 0001 = Type: postNATSourceIPv4Address (225)
|
74
|
+
Length: 4
|
75
|
+
Field (14/16): postNATDestinationIPv4Address
|
76
|
+
0... .... .... .... = Pen provided: No
|
77
|
+
.000 0000 1110 0010 = Type: postNATDestinationIPv4Address (226)
|
78
|
+
Length: 4
|
79
|
+
Field (15/16): postNAPTSourceTransportPort
|
80
|
+
0... .... .... .... = Pen provided: No
|
81
|
+
.000 0000 1110 0011 = Type: postNAPTSourceTransportPort (227)
|
82
|
+
Length: 2
|
83
|
+
Field (16/16): postNAPTDestinationTransportPort
|
84
|
+
0... .... .... .... = Pen provided: No
|
85
|
+
.000 0000 1110 0100 = Type: postNAPTDestinationTransportPort (228)
|
86
|
+
Length: 2
|
87
|
+
'''
|
88
|
+
|
89
|
+
data = "000a011d5b6b86c50000acf0a07e8c000100010d0010d49a000002fa06acd9022501bb00000002a3290a0000ed000000010a00000100dedac800debb88acd90225c0a8a84101bbdd690000009d00000001114b4b4b4b003500000002222c0a0000ed000000010a00000100de715000de71504b4b4b4bc0a8a8410035398f0000024600000005114a7d8a7101bb00000002d3d10a0000ed000000010a00000100de715000de71504a7d8a71c0a8a84101bb9ca40000038300000004114a7d8a7101bb0000000268920a0000ed000000010a00000100de715000de71504a7d8a71c0a8a84101bba46b000001c6000000040623a8ede501bb000000023d1b0a0000ed000000010a00000100de753800023a5023a8ede5c0a8a84101bbeb10".decode("hex")
|
90
|
+
|
91
|
+
'''
|
92
|
+
Cisco NetFlow/IPFIX
|
93
|
+
Version: 10
|
94
|
+
Length: 285
|
95
|
+
Timestamp: Aug 8, 2018 18:11:49.000000000 MDT
|
96
|
+
ExportTime: 1533773509
|
97
|
+
FlowSequence: 44272
|
98
|
+
Observation Domain Id: 2692647936
|
99
|
+
Set 1 [id=256] (5 flows)
|
100
|
+
FlowSet Id: (Data) (256)
|
101
|
+
FlowSet Length: 269
|
102
|
+
[Template Frame: 54]
|
103
|
+
Flow 1
|
104
|
+
Octets: 1103002
|
105
|
+
Packets: 762
|
106
|
+
Protocol: TCP (6)
|
107
|
+
SrcAddr: 172.217.2.37
|
108
|
+
SrcPort: 443 (443)
|
109
|
+
InputInt: 2
|
110
|
+
DstPort: 41769 (41769)
|
111
|
+
DstAddr: 10.0.0.237
|
112
|
+
OutputInt: 1
|
113
|
+
NextHop: 10.0.0.1
|
114
|
+
[Duration: 8.000000000 seconds (switched)]
|
115
|
+
StartTime: 14597.000000000 seconds
|
116
|
+
EndTime: 14605.000000000 seconds
|
117
|
+
Post NAT Source IPv4 Address: 172.217.2.37
|
118
|
+
Post NAT Destination IPv4 Address: 192.168.168.65
|
119
|
+
Post NAPT Source Transport Port: 443
|
120
|
+
Post NAPT Destination Transport Port: 56681
|
121
|
+
Flow 2
|
122
|
+
Octets: 157
|
123
|
+
Packets: 1
|
124
|
+
Protocol: UDP (17)
|
125
|
+
SrcAddr: 75.75.75.75
|
126
|
+
SrcPort: 53 (53)
|
127
|
+
InputInt: 2
|
128
|
+
DstPort: 8748 (8748)
|
129
|
+
DstAddr: 10.0.0.237
|
130
|
+
OutputInt: 1
|
131
|
+
NextHop: 10.0.0.1
|
132
|
+
[Duration: 0.000000000 seconds (switched)]
|
133
|
+
StartTime: 14578.000000000 seconds
|
134
|
+
EndTime: 14578.000000000 seconds
|
135
|
+
Post NAT Source IPv4 Address: 75.75.75.75
|
136
|
+
Post NAT Destination IPv4 Address: 192.168.168.65
|
137
|
+
Post NAPT Source Transport Port: 53
|
138
|
+
Post NAPT Destination Transport Port: 14735
|
139
|
+
Flow 3
|
140
|
+
Octets: 582
|
141
|
+
Packets: 5
|
142
|
+
Protocol: UDP (17)
|
143
|
+
SrcAddr: 74.125.138.113
|
144
|
+
SrcPort: 443 (443)
|
145
|
+
InputInt: 2
|
146
|
+
DstPort: 54225 (54225)
|
147
|
+
DstAddr: 10.0.0.237
|
148
|
+
OutputInt: 1
|
149
|
+
NextHop: 10.0.0.1
|
150
|
+
[Duration: 0.000000000 seconds (switched)]
|
151
|
+
StartTime: 14578.000000000 seconds
|
152
|
+
EndTime: 14578.000000000 seconds
|
153
|
+
Post NAT Source IPv4 Address: 74.125.138.113
|
154
|
+
Post NAT Destination IPv4 Address: 192.168.168.65
|
155
|
+
Post NAPT Source Transport Port: 443
|
156
|
+
Post NAPT Destination Transport Port: 40100
|
157
|
+
Flow 4
|
158
|
+
Octets: 899
|
159
|
+
Packets: 4
|
160
|
+
Protocol: UDP (17)
|
161
|
+
SrcAddr: 74.125.138.113
|
162
|
+
SrcPort: 443 (443)
|
163
|
+
InputInt: 2
|
164
|
+
DstPort: 26770 (26770)
|
165
|
+
DstAddr: 10.0.0.237
|
166
|
+
OutputInt: 1
|
167
|
+
NextHop: 10.0.0.1
|
168
|
+
[Duration: 0.000000000 seconds (switched)]
|
169
|
+
StartTime: 14578.000000000 seconds
|
170
|
+
EndTime: 14578.000000000 seconds
|
171
|
+
Post NAT Source IPv4 Address: 74.125.138.113
|
172
|
+
Post NAT Destination IPv4 Address: 192.168.168.65
|
173
|
+
Post NAPT Source Transport Port: 443
|
174
|
+
Post NAPT Destination Transport Port: 42091
|
175
|
+
Flow 5
|
176
|
+
Octets: 454
|
177
|
+
Packets: 4
|
178
|
+
Protocol: TCP (6)
|
179
|
+
SrcAddr: 35.168.237.229
|
180
|
+
SrcPort: 443 (443)
|
181
|
+
InputInt: 2
|
182
|
+
DstPort: 15643 (15643)
|
183
|
+
DstAddr: 10.0.0.237
|
184
|
+
OutputInt: 1
|
185
|
+
NextHop: 10.0.0.1
|
186
|
+
[Duration: 14433.000000000 seconds (switched)]
|
187
|
+
StartTime: 146.000000000 seconds
|
188
|
+
EndTime: 14579.000000000 seconds
|
189
|
+
Post NAT Source IPv4 Address: 35.168.237.229
|
190
|
+
Post NAT Destination IPv4 Address: 192.168.168.65
|
191
|
+
Post NAPT Source Transport Port: 443
|
192
|
+
Post NAPT Destination Transport Port: 60176
|
193
|
+
'''
|
194
|
+
|
195
|
+
host = sys.argv[1]
|
196
|
+
port = 2055
|
197
|
+
N = 150000
|
198
|
+
flowsPerPacket = 5
|
199
|
+
|
200
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
201
|
+
sock.sendto(tpl, (host, port))
|
202
|
+
time.sleep(0.2)
|
203
|
+
|
204
|
+
ts = time.time()
|
205
|
+
print("%d: started sending %d SonicWALL IPFIX flows in %d packets totaling %d bytes" % (ts,N*flowsPerPacket, N, N*len(data)))
|
206
|
+
print("%d: flow size %d, packet size %d" % (ts, len(data) / flowsPerPacket, len(data)))
|
207
|
+
|
208
|
+
for i in range(0, N):
|
209
|
+
sock.sendto(data, (host, port))
|