logstash-codec-sflow 1.1.1 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +24 -4
- data/lib/logstash/codecs/sflow.rb +17 -6
- data/lib/logstash/codecs/sflow/datagram.rb +2 -0
- data/lib/logstash/codecs/sflow/flow_record.rb +3 -0
- data/lib/logstash/codecs/sflow/packet_header.rb +0 -1
- data/lib/logstash/codecs/sflow/sample.rb +56 -1
- data/logstash-codec-sflow.gemspec +1 -1
- data/spec/codecs/sflow_spec.rb +4 -4
- metadata +17 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c28971730da8f9f302c926868c06bdac25a0d9b0
|
4
|
+
data.tar.gz: 6d114bd259a7194a9d61c8759c4c4bc2b400efce
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a6ef3af6d5a7f001997fa986c739eb0d2f23fb1857724ac4dbed38a211e69ca7f3518d778aed2e19d0e162a7c51695f02952d636bccf37bb4c22533e6221f0b7
|
7
|
+
data.tar.gz: 36e2350656250450e8d3299062e299c1e959f30c13e08d1cde9ba96825be6fe919a7c9102a7d5f3286419abd7b70251a4faff639dfb1168e42aaab06976bde87
|
data/README.md
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
## Description
|
3
3
|
Logstash codec plugin to decode sflow codec.
|
4
4
|
|
5
|
-
This codec manage flow sample and counter flow
|
5
|
+
This codec manage flow sample, counter flow, expanded flow sample and expanded counter flow
|
6
6
|
|
7
|
-
For the flow sample it is able to decode Ethernet, 802.1Q VLAN, IPv4, UDP and TCP header
|
7
|
+
For the (expanded) flow sample it is able to decode Ethernet, 802.1Q VLAN, IPv4, UDP and TCP header
|
8
8
|
|
9
|
-
For the counter flow it is able to decode some records of type:
|
9
|
+
For the (expanded) counter flow it is able to decode some records of type:
|
10
10
|
|
11
11
|
- Generic Interface
|
12
12
|
- Ethernet Interface
|
@@ -21,6 +21,25 @@ If needed you can aks for some to be added.
|
|
21
21
|
Please provide a pcap file containing the sflow events of the counter/protocol
|
22
22
|
to add in order to be able to implement it.
|
23
23
|
|
24
|
+
## Tune reported fields
|
25
|
+
By default all those fields are removed from the emitted event:
|
26
|
+
|
27
|
+
%w(sflow_version header_size ip_header_length ip_dscp ip_ecn ip_total_length ip_identification ip_flags
|
28
|
+
ip_fragment_offset ip_ttl ip_checksum ip_options tcp_seq_number tcp_ack_number tcp_header_length tcp_reserved
|
29
|
+
tcp_is_nonce tcp_is_cwr tcp_is_ecn_echo tcp_is_urgent tcp_is_ack tcp_is_push tcp_is_reset tcp_is_syn tcp_is_fin
|
30
|
+
tcp_window_size tcp_checksum tcp_urgent_pointer tcp_options vlan_cfi sequence_number flow_sequence_number vlan_type
|
31
|
+
udp_length udp_checksum)
|
32
|
+
|
33
|
+
You can tune the list of removed fields by setting this parameter to the sflow codec *optional_removed_field*
|
34
|
+
|
35
|
+
## frame_length_times_sampling_rate output field on (expanded) flow sample
|
36
|
+
|
37
|
+
This field is the length of the frame times the sampling rate. It permits to approximate the number of bits send/receive
|
38
|
+
on an interface/socket.
|
39
|
+
|
40
|
+
You must first ensure to have well configured the sampling rate to have an accurate output metric (See: http://blog.sflow.com/2009/06/sampling-rates.html)
|
41
|
+
|
42
|
+
|
24
43
|
## Human Readable Protocol
|
25
44
|
In order to translate protocols value to a human readable protocol, you can use the
|
26
45
|
logstash-filter-translate plugin
|
@@ -55,7 +74,8 @@ filter {
|
|
55
74
|
translate {
|
56
75
|
field => ip_protocol
|
57
76
|
dictionary => [ "6", "TCP",
|
58
|
-
"17", "UDP"
|
77
|
+
"17", "UDP",
|
78
|
+
"50", "Encapsulating Security Payload"
|
59
79
|
]
|
60
80
|
fallback => "UNKNOWN"
|
61
81
|
destination => ip_protocol
|
@@ -107,16 +107,18 @@ class LogStash::Codecs::Sflow < LogStash::Codecs::Base
|
|
107
107
|
events = []
|
108
108
|
|
109
109
|
decoded['samples'].each do |sample|
|
110
|
+
@logger.debug("sample: #{sample}")
|
110
111
|
#Treat case with no flow decoded (Unknown flow)
|
111
112
|
if sample['sample_data'].to_s.eql? ''
|
112
113
|
@logger.warn("Unknown sample entreprise #{sample['sample_entreprise'].to_s} - format #{sample['sample_format'].to_s}")
|
113
114
|
next
|
114
115
|
end
|
115
116
|
|
116
|
-
#treat sample flow
|
117
|
-
if sample['sample_entreprise'] == 0 && sample['sample_format'] == 1
|
117
|
+
#treat sample flow and expanded sample flow
|
118
|
+
if sample['sample_entreprise'] == 0 && (sample['sample_format'] == 1 || sample['sample_format'] == 3)
|
118
119
|
# Create the logstash event
|
119
120
|
event = LogStash::Event.new({})
|
121
|
+
|
120
122
|
common_sflow(event, decoded, sample)
|
121
123
|
|
122
124
|
sample['sample_data']['records'].each do |record|
|
@@ -134,15 +136,19 @@ class LogStash::Codecs::Sflow < LogStash::Codecs::Base
|
|
134
136
|
event["frame_length_times_sampling_rate"] = event['frame_length'].to_i * event['sampling_rate'].to_i
|
135
137
|
end
|
136
138
|
|
137
|
-
|
139
|
+
if sample['sample_format'] == 1
|
140
|
+
event["sflow_type"] = 'flow_sample'
|
141
|
+
else
|
142
|
+
event["sflow_type"] = 'expanded_flow_sample'
|
143
|
+
end
|
138
144
|
|
139
145
|
#Get interface dfescr if snmp_interface true
|
140
146
|
snmp_call(event)
|
141
147
|
|
142
148
|
events.push(event)
|
143
149
|
|
144
|
-
#treat counter flow
|
145
|
-
elsif sample['sample_entreprise'] == 0 && sample['sample_format'] == 2
|
150
|
+
#treat counter flow and expanded counter flow
|
151
|
+
elsif sample['sample_entreprise'] == 0 && (sample['sample_format'] == 2 || sample['sample_format'] == 4)
|
146
152
|
sample['sample_data']['records'].each do |record|
|
147
153
|
# Ensure that some data exist for the record
|
148
154
|
if record['record_data'].to_s.eql? ''
|
@@ -156,7 +162,12 @@ class LogStash::Codecs::Sflow < LogStash::Codecs::Base
|
|
156
162
|
|
157
163
|
assign_key_value(event, record)
|
158
164
|
|
159
|
-
|
165
|
+
if sample['sample_format'] == 2
|
166
|
+
event["sflow_type"] = 'counter_sample'
|
167
|
+
else
|
168
|
+
event["sflow_type"] = 'expanded_counter_sample'
|
169
|
+
end
|
170
|
+
|
160
171
|
|
161
172
|
#Get interface dfescr if snmp_interface true
|
162
173
|
snmp_call(event)
|
@@ -30,6 +30,8 @@ class SFlow < BinData::Record
|
|
30
30
|
choice :sample_data, :selection => lambda { "#{sample_entreprise}-#{sample_format}" } do
|
31
31
|
flow_sample '0-1'
|
32
32
|
counter_sample '0-2'
|
33
|
+
expanded_flow_sample '0-3'
|
34
|
+
expanded_counter_sample '0-4'
|
33
35
|
skip :default, :length => :sample_length
|
34
36
|
end
|
35
37
|
end
|
@@ -6,6 +6,8 @@ require 'logstash/codecs/sflow/packet_header'
|
|
6
6
|
|
7
7
|
# noinspection RubyResolve
|
8
8
|
class RawPacketHeader < BinData::Record
|
9
|
+
mandatory_parameter :record_length
|
10
|
+
|
9
11
|
endian :big
|
10
12
|
uint32 :protocol
|
11
13
|
uint32 :frame_length
|
@@ -16,6 +18,7 @@ class RawPacketHeader < BinData::Record
|
|
16
18
|
ip_header 11, :size_header => lambda { header_size * 8 }
|
17
19
|
skip :default, :length => :header_size
|
18
20
|
end
|
21
|
+
bit :padded, :nbits => lambda { (record_length - (header_size + 16)) * 8 } #padded data
|
19
22
|
end
|
20
23
|
|
21
24
|
# noinspection RubyResolve
|
@@ -21,7 +21,7 @@ class FlowSample < BinData::Record
|
|
21
21
|
bit12 :record_format
|
22
22
|
uint32 :record_length
|
23
23
|
choice :record_data, :selection => lambda { "#{record_entreprise}-#{record_format}" } do
|
24
|
-
raw_packet_header '0-1'
|
24
|
+
raw_packet_header '0-1', :record_length => :record_length
|
25
25
|
ethernet_frame_data '0-2'
|
26
26
|
ip4_data '0-3'
|
27
27
|
ip6_data '0-4'
|
@@ -56,3 +56,58 @@ class CounterSample < BinData::Record
|
|
56
56
|
#processor_information :record_data
|
57
57
|
end
|
58
58
|
end
|
59
|
+
|
60
|
+
# noinspection RubyResolve
|
61
|
+
class ExpandedFlowSample < BinData::Record
|
62
|
+
endian :big
|
63
|
+
uint32 :flow_sequence_number
|
64
|
+
uint32 :source_id_type
|
65
|
+
uint32 :source_id_index
|
66
|
+
uint32 :sampling_rate
|
67
|
+
uint32 :sample_pool
|
68
|
+
uint32 :drops
|
69
|
+
uint32 :input_interface_format
|
70
|
+
uint32 :input_interface_value
|
71
|
+
uint32 :output_interface_format
|
72
|
+
uint32 :output_interface_value
|
73
|
+
uint32 :record_count
|
74
|
+
array :records, :initial_length => :record_count do
|
75
|
+
bit20 :record_entreprise
|
76
|
+
bit12 :record_format
|
77
|
+
uint32 :record_length
|
78
|
+
choice :record_data, :selection => lambda { "#{record_entreprise}-#{record_format}" } do
|
79
|
+
raw_packet_header '0-1', :record_length => :record_length
|
80
|
+
ethernet_frame_data '0-2'
|
81
|
+
ip4_data '0-3'
|
82
|
+
ip6_data '0-4'
|
83
|
+
extended_switch_data '0-1001'
|
84
|
+
extended_router_data '0-1002'
|
85
|
+
skip :default, :length => :record_length
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
# noinspection RubyResolve
|
91
|
+
class ExpandedCounterSample < BinData::Record
|
92
|
+
endian :big
|
93
|
+
uint32 :sample_seq_number
|
94
|
+
uint32 :source_id_type
|
95
|
+
uint32 :source_id_index
|
96
|
+
uint32 :record_count
|
97
|
+
array :records, :initial_length => :record_count do
|
98
|
+
bit20 :record_entreprise
|
99
|
+
bit12 :record_format
|
100
|
+
uint32 :record_length
|
101
|
+
choice :record_data, :selection => lambda { "#{record_entreprise}-#{record_format}" } do
|
102
|
+
generic_interface '0-1'
|
103
|
+
ethernet_interfaces '0-2'
|
104
|
+
token_ring '0-3'
|
105
|
+
hundred_base_vg '0-4'
|
106
|
+
vlan '0-5'
|
107
|
+
processor_information '0-1001'
|
108
|
+
http_counters '0-2201'
|
109
|
+
skip :default, :length => :record_length
|
110
|
+
end
|
111
|
+
#processor_information :record_data
|
112
|
+
end
|
113
|
+
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-codec-sflow'
|
4
|
-
s.version = '1.
|
4
|
+
s.version = '1.2.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = 'The sflow codec is for decoding SFlow v5 flows.'
|
7
7
|
s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'
|
data/spec/codecs/sflow_spec.rb
CHANGED
@@ -7,21 +7,21 @@ require "logstash/codecs/sflow/datagram"
|
|
7
7
|
describe SFlow do
|
8
8
|
it "should decode sflow counters" do
|
9
9
|
payload = IO.read(File.join(File.dirname(__FILE__), "sflow_counters_sample.dat"), :mode => "rb")
|
10
|
-
|
10
|
+
SFlow.read(payload)
|
11
11
|
end
|
12
12
|
|
13
13
|
it "should decode sflow 1 counters" do
|
14
14
|
payload = IO.read(File.join(File.dirname(__FILE__), "sflow_1_counters_sample.dat"), :mode => "rb")
|
15
|
-
|
15
|
+
SFlow.read(payload)
|
16
16
|
end
|
17
17
|
|
18
18
|
it "should decode sflow sample" do
|
19
19
|
payload = IO.read(File.join(File.dirname(__FILE__), "sflow_flow_sample.dat"), :mode => "rb")
|
20
|
-
|
20
|
+
SFlow.read(payload)
|
21
21
|
end
|
22
22
|
|
23
23
|
it "should decode sflow sample eth vlan" do
|
24
24
|
payload = IO.read(File.join(File.dirname(__FILE__), "sflow_flow_sample_eth_vlan.dat"), :mode => "rb")
|
25
|
-
|
25
|
+
SFlow.read(payload)
|
26
26
|
end
|
27
27
|
end
|
metadata
CHANGED
@@ -1,22 +1,22 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-sflow
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nicolas Fraison
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-08-
|
11
|
+
date: 2016-08-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
15
15
|
requirements:
|
16
|
-
- -
|
16
|
+
- - ">="
|
17
17
|
- !ruby/object:Gem::Version
|
18
18
|
version: 1.4.0
|
19
|
-
- - <
|
19
|
+
- - "<"
|
20
20
|
- !ruby/object:Gem::Version
|
21
21
|
version: 3.0.0
|
22
22
|
name: logstash-core
|
@@ -24,16 +24,16 @@ dependencies:
|
|
24
24
|
type: :runtime
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
|
-
- -
|
27
|
+
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: 1.4.0
|
30
|
-
- - <
|
30
|
+
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: 3.0.0
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
34
|
requirement: !ruby/object:Gem::Requirement
|
35
35
|
requirements:
|
36
|
-
- -
|
36
|
+
- - ">="
|
37
37
|
- !ruby/object:Gem::Version
|
38
38
|
version: 2.3.0
|
39
39
|
name: bindata
|
@@ -41,13 +41,13 @@ dependencies:
|
|
41
41
|
type: :runtime
|
42
42
|
version_requirements: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
44
|
-
- -
|
44
|
+
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
46
|
version: 2.3.0
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
49
49
|
requirements:
|
50
|
-
- -
|
50
|
+
- - ">="
|
51
51
|
- !ruby/object:Gem::Version
|
52
52
|
version: 1.1.0
|
53
53
|
name: lru_redux
|
@@ -55,13 +55,13 @@ dependencies:
|
|
55
55
|
type: :runtime
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
|
-
- -
|
58
|
+
- - ">="
|
59
59
|
- !ruby/object:Gem::Version
|
60
60
|
version: 1.1.0
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
requirement: !ruby/object:Gem::Requirement
|
63
63
|
requirements:
|
64
|
-
- -
|
64
|
+
- - ">="
|
65
65
|
- !ruby/object:Gem::Version
|
66
66
|
version: 1.2.0
|
67
67
|
name: snmp
|
@@ -69,13 +69,13 @@ dependencies:
|
|
69
69
|
type: :runtime
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
71
71
|
requirements:
|
72
|
-
- -
|
72
|
+
- - ">="
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: 1.2.0
|
75
75
|
- !ruby/object:Gem::Dependency
|
76
76
|
requirement: !ruby/object:Gem::Requirement
|
77
77
|
requirements:
|
78
|
-
- -
|
78
|
+
- - ">="
|
79
79
|
- !ruby/object:Gem::Version
|
80
80
|
version: '0'
|
81
81
|
name: logstash-devutils
|
@@ -83,7 +83,7 @@ dependencies:
|
|
83
83
|
type: :development
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
85
85
|
requirements:
|
86
|
-
- -
|
86
|
+
- - ">="
|
87
87
|
- !ruby/object:Gem::Version
|
88
88
|
version: '0'
|
89
89
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
@@ -130,17 +130,17 @@ require_paths:
|
|
130
130
|
- lib
|
131
131
|
required_ruby_version: !ruby/object:Gem::Requirement
|
132
132
|
requirements:
|
133
|
-
- -
|
133
|
+
- - ">="
|
134
134
|
- !ruby/object:Gem::Version
|
135
135
|
version: '0'
|
136
136
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
137
137
|
requirements:
|
138
|
-
- -
|
138
|
+
- - ">="
|
139
139
|
- !ruby/object:Gem::Version
|
140
140
|
version: '0'
|
141
141
|
requirements: []
|
142
142
|
rubyforge_project:
|
143
|
-
rubygems_version: 2.4.
|
143
|
+
rubygems_version: 2.4.8
|
144
144
|
signing_key:
|
145
145
|
specification_version: 4
|
146
146
|
summary: The sflow codec is for decoding SFlow v5 flows.
|