logstash-codec-sflow 1.1.1 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ddc03024121f42f455dd1275cfe5cc6413f5b203
4
- data.tar.gz: 84f0a827141d0fe80c01e1f8b373a649229be489
3
+ metadata.gz: c28971730da8f9f302c926868c06bdac25a0d9b0
4
+ data.tar.gz: 6d114bd259a7194a9d61c8759c4c4bc2b400efce
5
5
  SHA512:
6
- metadata.gz: 49681626b90b77e12f4091c96c1bd7cedbee381c15be3a13cb56a20796c1df6b061de9e3c77799087535d0959eb1789b758670e13e0997f5dd711154c9d8760f
7
- data.tar.gz: dcdbcc07970c0c8c15960676a3925e6bebd070a2f0b6c084e4011301343bf9de0b8ed071f72e288a9e90a99d5dd4de32946e365470030cf5de85611489304961
6
+ metadata.gz: a6ef3af6d5a7f001997fa986c739eb0d2f23fb1857724ac4dbed38a211e69ca7f3518d778aed2e19d0e162a7c51695f02952d636bccf37bb4c22533e6221f0b7
7
+ data.tar.gz: 36e2350656250450e8d3299062e299c1e959f30c13e08d1cde9ba96825be6fe919a7c9102a7d5f3286419abd7b70251a4faff639dfb1168e42aaab06976bde87
data/README.md CHANGED
@@ -2,11 +2,11 @@
2
2
  ## Description
3
3
  Logstash codec plugin to decode sflow codec.
4
4
 
5
- This codec manage flow sample and counter flow.
5
+ This codec manage flow sample, counter flow, expanded flow sample and expanded counter flow
6
6
 
7
- For the flow sample it is able to decode Ethernet, 802.1Q VLAN, IPv4, UDP and TCP header
7
+ For the (expanded) flow sample it is able to decode Ethernet, 802.1Q VLAN, IPv4, UDP and TCP header
8
8
 
9
- For the counter flow it is able to decode some records of type:
9
+ For the (expanded) counter flow it is able to decode some records of type:
10
10
 
11
11
  - Generic Interface
12
12
  - Ethernet Interface
@@ -21,6 +21,25 @@ If needed you can aks for some to be added.
21
21
  Please provide a pcap file containing the sflow events of the counter/protocol
22
22
  to add in order to be able to implement it.
23
23
 
24
+ ## Tune reported fields
25
+ By default all those fields are removed from the emitted event:
26
+
27
+ %w(sflow_version header_size ip_header_length ip_dscp ip_ecn ip_total_length ip_identification ip_flags
28
+ ip_fragment_offset ip_ttl ip_checksum ip_options tcp_seq_number tcp_ack_number tcp_header_length tcp_reserved
29
+ tcp_is_nonce tcp_is_cwr tcp_is_ecn_echo tcp_is_urgent tcp_is_ack tcp_is_push tcp_is_reset tcp_is_syn tcp_is_fin
30
+ tcp_window_size tcp_checksum tcp_urgent_pointer tcp_options vlan_cfi sequence_number flow_sequence_number vlan_type
31
+ udp_length udp_checksum)
32
+
33
+ You can tune the list of removed fields by setting this parameter to the sflow codec *optional_removed_field*
34
+
35
+ ## frame_length_times_sampling_rate output field on (expanded) flow sample
36
+
37
+ This field is the length of the frame times the sampling rate. It permits to approximate the number of bits send/receive
38
+ on an interface/socket.
39
+
40
+ You must first ensure to have well configured the sampling rate to have an accurate output metric (See: http://blog.sflow.com/2009/06/sampling-rates.html)
41
+
42
+
24
43
  ## Human Readable Protocol
25
44
  In order to translate protocols value to a human readable protocol, you can use the
26
45
  logstash-filter-translate plugin
@@ -55,7 +74,8 @@ filter {
55
74
  translate {
56
75
  field => ip_protocol
57
76
  dictionary => [ "6", "TCP",
58
- "17", "UDP"
77
+ "17", "UDP",
78
+ "50", "Encapsulating Security Payload"
59
79
  ]
60
80
  fallback => "UNKNOWN"
61
81
  destination => ip_protocol
@@ -107,16 +107,18 @@ class LogStash::Codecs::Sflow < LogStash::Codecs::Base
107
107
  events = []
108
108
 
109
109
  decoded['samples'].each do |sample|
110
+ @logger.debug("sample: #{sample}")
110
111
  #Treat case with no flow decoded (Unknown flow)
111
112
  if sample['sample_data'].to_s.eql? ''
112
113
  @logger.warn("Unknown sample entreprise #{sample['sample_entreprise'].to_s} - format #{sample['sample_format'].to_s}")
113
114
  next
114
115
  end
115
116
 
116
- #treat sample flow
117
- if sample['sample_entreprise'] == 0 && sample['sample_format'] == 1
117
+ #treat sample flow and expanded sample flow
118
+ if sample['sample_entreprise'] == 0 && (sample['sample_format'] == 1 || sample['sample_format'] == 3)
118
119
  # Create the logstash event
119
120
  event = LogStash::Event.new({})
121
+
120
122
  common_sflow(event, decoded, sample)
121
123
 
122
124
  sample['sample_data']['records'].each do |record|
@@ -134,15 +136,19 @@ class LogStash::Codecs::Sflow < LogStash::Codecs::Base
134
136
  event["frame_length_times_sampling_rate"] = event['frame_length'].to_i * event['sampling_rate'].to_i
135
137
  end
136
138
 
137
- event["sflow_type"] = 'sample'
139
+ if sample['sample_format'] == 1
140
+ event["sflow_type"] = 'flow_sample'
141
+ else
142
+ event["sflow_type"] = 'expanded_flow_sample'
143
+ end
138
144
 
139
145
  #Get interface dfescr if snmp_interface true
140
146
  snmp_call(event)
141
147
 
142
148
  events.push(event)
143
149
 
144
- #treat counter flow
145
- elsif sample['sample_entreprise'] == 0 && sample['sample_format'] == 2
150
+ #treat counter flow and expanded counter flow
151
+ elsif sample['sample_entreprise'] == 0 && (sample['sample_format'] == 2 || sample['sample_format'] == 4)
146
152
  sample['sample_data']['records'].each do |record|
147
153
  # Ensure that some data exist for the record
148
154
  if record['record_data'].to_s.eql? ''
@@ -156,7 +162,12 @@ class LogStash::Codecs::Sflow < LogStash::Codecs::Base
156
162
 
157
163
  assign_key_value(event, record)
158
164
 
159
- event["sflow_type"] = 'counter'
165
+ if sample['sample_format'] == 2
166
+ event["sflow_type"] = 'counter_sample'
167
+ else
168
+ event["sflow_type"] = 'expanded_counter_sample'
169
+ end
170
+
160
171
 
161
172
  #Get interface dfescr if snmp_interface true
162
173
  snmp_call(event)
@@ -30,6 +30,8 @@ class SFlow < BinData::Record
30
30
  choice :sample_data, :selection => lambda { "#{sample_entreprise}-#{sample_format}" } do
31
31
  flow_sample '0-1'
32
32
  counter_sample '0-2'
33
+ expanded_flow_sample '0-3'
34
+ expanded_counter_sample '0-4'
33
35
  skip :default, :length => :sample_length
34
36
  end
35
37
  end
@@ -6,6 +6,8 @@ require 'logstash/codecs/sflow/packet_header'
6
6
 
7
7
  # noinspection RubyResolve
8
8
  class RawPacketHeader < BinData::Record
9
+ mandatory_parameter :record_length
10
+
9
11
  endian :big
10
12
  uint32 :protocol
11
13
  uint32 :frame_length
@@ -16,6 +18,7 @@ class RawPacketHeader < BinData::Record
16
18
  ip_header 11, :size_header => lambda { header_size * 8 }
17
19
  skip :default, :length => :header_size
18
20
  end
21
+ bit :padded, :nbits => lambda { (record_length - (header_size + 16)) * 8 } #padded data
19
22
  end
20
23
 
21
24
  # noinspection RubyResolve
@@ -55,7 +55,6 @@ class UdpHeader < BinData::Record
55
55
  uint16 :dst_port
56
56
  uint16 :udp_length
57
57
  uint16 :udp_checksum
58
- #skip :length => lambda { udp_length - 64 } #skip udp data
59
58
  bit :data, :nbits => lambda { size_header - 64 } #skip udp data
60
59
  end
61
60
 
@@ -21,7 +21,7 @@ class FlowSample < BinData::Record
21
21
  bit12 :record_format
22
22
  uint32 :record_length
23
23
  choice :record_data, :selection => lambda { "#{record_entreprise}-#{record_format}" } do
24
- raw_packet_header '0-1'
24
+ raw_packet_header '0-1', :record_length => :record_length
25
25
  ethernet_frame_data '0-2'
26
26
  ip4_data '0-3'
27
27
  ip6_data '0-4'
@@ -56,3 +56,58 @@ class CounterSample < BinData::Record
56
56
  #processor_information :record_data
57
57
  end
58
58
  end
59
+
60
+ # noinspection RubyResolve
61
+ class ExpandedFlowSample < BinData::Record
62
+ endian :big
63
+ uint32 :flow_sequence_number
64
+ uint32 :source_id_type
65
+ uint32 :source_id_index
66
+ uint32 :sampling_rate
67
+ uint32 :sample_pool
68
+ uint32 :drops
69
+ uint32 :input_interface_format
70
+ uint32 :input_interface_value
71
+ uint32 :output_interface_format
72
+ uint32 :output_interface_value
73
+ uint32 :record_count
74
+ array :records, :initial_length => :record_count do
75
+ bit20 :record_entreprise
76
+ bit12 :record_format
77
+ uint32 :record_length
78
+ choice :record_data, :selection => lambda { "#{record_entreprise}-#{record_format}" } do
79
+ raw_packet_header '0-1', :record_length => :record_length
80
+ ethernet_frame_data '0-2'
81
+ ip4_data '0-3'
82
+ ip6_data '0-4'
83
+ extended_switch_data '0-1001'
84
+ extended_router_data '0-1002'
85
+ skip :default, :length => :record_length
86
+ end
87
+ end
88
+ end
89
+
90
+ # noinspection RubyResolve
91
+ class ExpandedCounterSample < BinData::Record
92
+ endian :big
93
+ uint32 :sample_seq_number
94
+ uint32 :source_id_type
95
+ uint32 :source_id_index
96
+ uint32 :record_count
97
+ array :records, :initial_length => :record_count do
98
+ bit20 :record_entreprise
99
+ bit12 :record_format
100
+ uint32 :record_length
101
+ choice :record_data, :selection => lambda { "#{record_entreprise}-#{record_format}" } do
102
+ generic_interface '0-1'
103
+ ethernet_interfaces '0-2'
104
+ token_ring '0-3'
105
+ hundred_base_vg '0-4'
106
+ vlan '0-5'
107
+ processor_information '0-1001'
108
+ http_counters '0-2201'
109
+ skip :default, :length => :record_length
110
+ end
111
+ #processor_information :record_data
112
+ end
113
+ end
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-sflow'
4
- s.version = '1.1.1'
4
+ s.version = '1.2.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = 'The sflow codec is for decoding SFlow v5 flows.'
7
7
  s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'
@@ -7,21 +7,21 @@ require "logstash/codecs/sflow/datagram"
7
7
  describe SFlow do
8
8
  it "should decode sflow counters" do
9
9
  payload = IO.read(File.join(File.dirname(__FILE__), "sflow_counters_sample.dat"), :mode => "rb")
10
- decoded = SFlow.read(payload)
10
+ SFlow.read(payload)
11
11
  end
12
12
 
13
13
  it "should decode sflow 1 counters" do
14
14
  payload = IO.read(File.join(File.dirname(__FILE__), "sflow_1_counters_sample.dat"), :mode => "rb")
15
- decoded = SFlow.read(payload)
15
+ SFlow.read(payload)
16
16
  end
17
17
 
18
18
  it "should decode sflow sample" do
19
19
  payload = IO.read(File.join(File.dirname(__FILE__), "sflow_flow_sample.dat"), :mode => "rb")
20
- decoded = SFlow.read(payload)
20
+ SFlow.read(payload)
21
21
  end
22
22
 
23
23
  it "should decode sflow sample eth vlan" do
24
24
  payload = IO.read(File.join(File.dirname(__FILE__), "sflow_flow_sample_eth_vlan.dat"), :mode => "rb")
25
- decoded = SFlow.read(payload)
25
+ SFlow.read(payload)
26
26
  end
27
27
  end
metadata CHANGED
@@ -1,22 +1,22 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-sflow
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.1
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nicolas Fraison
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-08-01 00:00:00.000000000 Z
11
+ date: 2016-08-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
- - - '>='
16
+ - - ">="
17
17
  - !ruby/object:Gem::Version
18
18
  version: 1.4.0
19
- - - <
19
+ - - "<"
20
20
  - !ruby/object:Gem::Version
21
21
  version: 3.0.0
22
22
  name: logstash-core
@@ -24,16 +24,16 @@ dependencies:
24
24
  type: :runtime
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
27
- - - '>='
27
+ - - ">="
28
28
  - !ruby/object:Gem::Version
29
29
  version: 1.4.0
30
- - - <
30
+ - - "<"
31
31
  - !ruby/object:Gem::Version
32
32
  version: 3.0.0
33
33
  - !ruby/object:Gem::Dependency
34
34
  requirement: !ruby/object:Gem::Requirement
35
35
  requirements:
36
- - - '>='
36
+ - - ">="
37
37
  - !ruby/object:Gem::Version
38
38
  version: 2.3.0
39
39
  name: bindata
@@ -41,13 +41,13 @@ dependencies:
41
41
  type: :runtime
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
- - - '>='
44
+ - - ">="
45
45
  - !ruby/object:Gem::Version
46
46
  version: 2.3.0
47
47
  - !ruby/object:Gem::Dependency
48
48
  requirement: !ruby/object:Gem::Requirement
49
49
  requirements:
50
- - - '>='
50
+ - - ">="
51
51
  - !ruby/object:Gem::Version
52
52
  version: 1.1.0
53
53
  name: lru_redux
@@ -55,13 +55,13 @@ dependencies:
55
55
  type: :runtime
56
56
  version_requirements: !ruby/object:Gem::Requirement
57
57
  requirements:
58
- - - '>='
58
+ - - ">="
59
59
  - !ruby/object:Gem::Version
60
60
  version: 1.1.0
61
61
  - !ruby/object:Gem::Dependency
62
62
  requirement: !ruby/object:Gem::Requirement
63
63
  requirements:
64
- - - '>='
64
+ - - ">="
65
65
  - !ruby/object:Gem::Version
66
66
  version: 1.2.0
67
67
  name: snmp
@@ -69,13 +69,13 @@ dependencies:
69
69
  type: :runtime
70
70
  version_requirements: !ruby/object:Gem::Requirement
71
71
  requirements:
72
- - - '>='
72
+ - - ">="
73
73
  - !ruby/object:Gem::Version
74
74
  version: 1.2.0
75
75
  - !ruby/object:Gem::Dependency
76
76
  requirement: !ruby/object:Gem::Requirement
77
77
  requirements:
78
- - - '>='
78
+ - - ">="
79
79
  - !ruby/object:Gem::Version
80
80
  version: '0'
81
81
  name: logstash-devutils
@@ -83,7 +83,7 @@ dependencies:
83
83
  type: :development
84
84
  version_requirements: !ruby/object:Gem::Requirement
85
85
  requirements:
86
- - - '>='
86
+ - - ">="
87
87
  - !ruby/object:Gem::Version
88
88
  version: '0'
89
89
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
@@ -130,17 +130,17 @@ require_paths:
130
130
  - lib
131
131
  required_ruby_version: !ruby/object:Gem::Requirement
132
132
  requirements:
133
- - - '>='
133
+ - - ">="
134
134
  - !ruby/object:Gem::Version
135
135
  version: '0'
136
136
  required_rubygems_version: !ruby/object:Gem::Requirement
137
137
  requirements:
138
- - - '>='
138
+ - - ">="
139
139
  - !ruby/object:Gem::Version
140
140
  version: '0'
141
141
  requirements: []
142
142
  rubyforge_project:
143
- rubygems_version: 2.4.5
143
+ rubygems_version: 2.4.8
144
144
  signing_key:
145
145
  specification_version: 4
146
146
  summary: The sflow codec is for decoding SFlow v5 flows.