fluent-plugin-sflow 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +18 -4
- data/README.md +254 -13
- data/Rakefile +10 -7
- data/example/fluentd.conf +4 -2
- data/ext/sflowtool/extconf.rb +27 -0
- data/ext/sflowtool/fluent-plugin-sflow.c +52 -0
- data/ext/sflowtool/sflow.h +1821 -0
- data/ext/sflowtool/sflow_v2v4.h +436 -0
- data/ext/sflowtool/sflowtool.c +4801 -0
- data/fluent-plugin-sflow.gemspec +11 -25
- data/lib/fluent/plugin/in_sflow.rb +66 -23
- data/lib/fluent/plugin/parser_sflow.rb +24 -0
- data/lib/sflowtool.rb +17 -0
- metadata +35 -84
- data/.gitmodules +0 -3
- data/LICENSE.txt +0 -21
- data/lib/sflow/Gemfile +0 -8
- data/lib/sflow/Gemfile.lock +0 -38
- data/lib/sflow/LICENSE.txt +0 -22
- data/lib/sflow/README.md +0 -67
- data/lib/sflow/Rakefile +0 -12
- data/lib/sflow/bin/bundler +0 -16
- data/lib/sflow/bin/rake +0 -16
- data/lib/sflow/bin/sflow.rb +0 -7
- data/lib/sflow/etc/config.yaml +0 -10
- data/lib/sflow/lib/sflow.rb +0 -10
- data/lib/sflow/lib/sflow/collector.rb +0 -69
- data/lib/sflow/lib/sflow/config.rb +0 -15
- data/lib/sflow/lib/sflow/models/binary_models.rb +0 -176
- data/lib/sflow/lib/sflow/models/ipv4header.rb +0 -69
- data/lib/sflow/lib/sflow/models/protocol.rb +0 -47
- data/lib/sflow/lib/sflow/models/tcpheader.rb +0 -82
- data/lib/sflow/lib/sflow/models/udpheader.rb +0 -36
- data/lib/sflow/lib/sflow/parsers/parsers.rb +0 -68
- data/lib/sflow/lib/sflow/snmp/iface_names.rb +0 -40
- data/lib/sflow/lib/sflow/storage/storage.rb +0 -34
- data/lib/sflow/lib/sflow/version.rb +0 -3
- data/lib/sflow/misc/kibana-schema.json +0 -1364
- data/lib/sflow/misc/screen1.png +0 -0
- data/lib/sflow/sflow.gemspec +0 -23
- data/lib/sflow/test/lib/sflow/version_test.rb +0 -8
- data/lib/sflow/test/test_helper.rb +0 -4
data/lib/sflow/Gemfile.lock
DELETED
@@ -1,38 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
sflow (0.0.1)
|
5
|
-
|
6
|
-
GEM
|
7
|
-
remote: https://rubygems.org/
|
8
|
-
specs:
|
9
|
-
bindata (1.8.1)
|
10
|
-
elasticsearch (1.0.1)
|
11
|
-
elasticsearch-api (= 1.0.1)
|
12
|
-
elasticsearch-transport (= 1.0.1)
|
13
|
-
elasticsearch-api (1.0.1)
|
14
|
-
multi_json
|
15
|
-
elasticsearch-transport (1.0.1)
|
16
|
-
faraday
|
17
|
-
multi_json
|
18
|
-
eventmachine (1.0.3)
|
19
|
-
faraday (0.8.8)
|
20
|
-
multipart-post (~> 1.2.0)
|
21
|
-
json (1.8.1)
|
22
|
-
minitest (5.3.4)
|
23
|
-
multi_json (1.7.9)
|
24
|
-
multipart-post (1.2.0)
|
25
|
-
rake (10.2.0)
|
26
|
-
|
27
|
-
PLATFORMS
|
28
|
-
ruby
|
29
|
-
|
30
|
-
DEPENDENCIES
|
31
|
-
bindata
|
32
|
-
bundler (~> 1.5)
|
33
|
-
elasticsearch
|
34
|
-
eventmachine
|
35
|
-
json
|
36
|
-
minitest
|
37
|
-
rake
|
38
|
-
sflow!
|
data/lib/sflow/LICENSE.txt
DELETED
@@ -1,22 +0,0 @@
|
|
1
|
-
Copyright (c) 2014 Sebastian Saemann <ssaemann@netways.de>
|
2
|
-
|
3
|
-
MIT License
|
4
|
-
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
-
a copy of this software and associated documentation files (the
|
7
|
-
"Software"), to deal in the Software without restriction, including
|
8
|
-
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
-
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
-
permit persons to whom the Software is furnished to do so, subject to
|
11
|
-
the following conditions:
|
12
|
-
|
13
|
-
The above copyright notice and this permission notice shall be
|
14
|
-
included in all copies or substantial portions of the Software.
|
15
|
-
|
16
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
-
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
-
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
-
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/lib/sflow/README.md
DELETED
@@ -1,67 +0,0 @@
|
|
1
|
-
# Sflow
|
2
|
-
|
3
|
-
Tiny sflow collector and parser script based on eventmachine. It listens for sflow v5 samples, parses them and sends it to logstash.
|
4
|
-
|
5
|
-
## Installation
|
6
|
-
|
7
|
-
Clone this repository
|
8
|
-
|
9
|
-
$ git clone http://github.com/netways/sflow
|
10
|
-
|
11
|
-
Change directory
|
12
|
-
|
13
|
-
$ cd sflow
|
14
|
-
|
15
|
-
Install dependencies using bundler
|
16
|
-
|
17
|
-
$ bundle install
|
18
|
-
|
19
|
-
Configure your logstash endpoint
|
20
|
-
|
21
|
-
$ vi ./etc/config.yaml
|
22
|
-
|
23
|
-
And then execute:
|
24
|
-
|
25
|
-
$ bundle exec ./bin/sflow.rb
|
26
|
-
|
27
|
-
## Logstash Configuration
|
28
|
-
|
29
|
-
A complete logstash installation is a prerequisite.
|
30
|
-
|
31
|
-
For getting the parsed sflow-packets as JSON via UDP into logstash you have to configure a input, filter and a output accordingly:
|
32
|
-
|
33
|
-
input {
|
34
|
-
udp {
|
35
|
-
port => 6543
|
36
|
-
type => "sflow"
|
37
|
-
codec => 'json'
|
38
|
-
}
|
39
|
-
}
|
40
|
-
|
41
|
-
filter {
|
42
|
-
json {
|
43
|
-
source => "message"
|
44
|
-
type => "json"
|
45
|
-
}
|
46
|
-
}
|
47
|
-
|
48
|
-
output {
|
49
|
-
elasticsearch_http {
|
50
|
-
workers => 8
|
51
|
-
host => "elasticsearch.host"
|
52
|
-
}
|
53
|
-
}
|
54
|
-
|
55
|
-
## Kibana
|
56
|
-
|
57
|
-
You can create your very own kibana dashboard for viewing the information and graphs you are interested in. For a quick start you'll find a dashboard in the misc folder, which can be imported via the kibana webinterface.
|
58
|
-
|
59
|
-

|
60
|
-
|
61
|
-
## Contributing
|
62
|
-
|
63
|
-
1. Fork it ( http://github.com/netways/sflow/fork )
|
64
|
-
2. Create your feature branch (`git checkout -b my-new-feature`)
|
65
|
-
3. Commit your changes (`git commit -am 'Add some feature'`)
|
66
|
-
4. Push to the branch (`git push origin my-new-feature`)
|
67
|
-
5. Create new Pull Request
|
data/lib/sflow/Rakefile
DELETED
data/lib/sflow/bin/bundler
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby1.9.1
|
2
|
-
#
|
3
|
-
# This file was generated by Bundler.
|
4
|
-
#
|
5
|
-
# The application 'bundler' is installed as part of a gem, and
|
6
|
-
# this file is here to facilitate running it.
|
7
|
-
#
|
8
|
-
|
9
|
-
require 'pathname'
|
10
|
-
ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
|
11
|
-
Pathname.new(__FILE__).realpath)
|
12
|
-
|
13
|
-
require 'rubygems'
|
14
|
-
require 'bundler/setup'
|
15
|
-
|
16
|
-
load Gem.bin_path('bundler', 'bundler')
|
data/lib/sflow/bin/rake
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby1.9.1
|
2
|
-
#
|
3
|
-
# This file was generated by Bundler.
|
4
|
-
#
|
5
|
-
# The application 'rake' is installed as part of a gem, and
|
6
|
-
# this file is here to facilitate running it.
|
7
|
-
#
|
8
|
-
|
9
|
-
require 'pathname'
|
10
|
-
ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
|
11
|
-
Pathname.new(__FILE__).realpath)
|
12
|
-
|
13
|
-
require 'rubygems'
|
14
|
-
require 'bundler/setup'
|
15
|
-
|
16
|
-
load Gem.bin_path('rake', 'rake')
|
data/lib/sflow/bin/sflow.rb
DELETED
data/lib/sflow/etc/config.yaml
DELETED
data/lib/sflow/lib/sflow.rb
DELETED
@@ -1,10 +0,0 @@
|
|
1
|
-
require 'bindata'
|
2
|
-
require 'eventmachine'
|
3
|
-
require 'yaml'
|
4
|
-
|
5
|
-
dir = File.expand_path(File.join(File.dirname(__FILE__), 'sflow'))
|
6
|
-
['config','models/ipv4header', 'models/tcpheader', 'models/udpheader', 'models/protocol', 'models/binary_models','parsers/parsers','storage/storage', 'collector','snmp/iface_names'].each do |req|
|
7
|
-
require File.join(dir, req)
|
8
|
-
end
|
9
|
-
|
10
|
-
Process.daemon(true) if $daemonize == true
|
@@ -1,69 +0,0 @@
|
|
1
|
-
class SflowCollector
|
2
|
-
module Collector
|
3
|
-
Thread.abort_on_exception=true
|
4
|
-
require 'socket'
|
5
|
-
def post_init
|
6
|
-
puts "Server listening."
|
7
|
-
end
|
8
|
-
|
9
|
-
def receive_data(data)
|
10
|
-
operation = proc do
|
11
|
-
begin
|
12
|
-
if data != nil
|
13
|
-
sflow = SflowParser.parse_packet(data)
|
14
|
-
end
|
15
|
-
rescue Exception => e
|
16
|
-
puts Time.now
|
17
|
-
puts sflow.inspect
|
18
|
-
puts e.message
|
19
|
-
puts e.backtrace
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
callback = proc do |sflow|
|
24
|
-
begin
|
25
|
-
if sflow != nil
|
26
|
-
SflowStorage.send_udpjson(sflow)
|
27
|
-
end
|
28
|
-
rescue Exception => e
|
29
|
-
puts Time.now
|
30
|
-
puts sflow.inspect if sflow != nil
|
31
|
-
puts e.message
|
32
|
-
puts e.backtrace
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
EM.defer(operation,callback)
|
37
|
-
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
def self.start_collector(bind_ip = '0.0.0.0', bind_port = 6343)
|
42
|
-
begin
|
43
|
-
config = SflowConfig.new
|
44
|
-
if config.logstash_host and config.logstash_port
|
45
|
-
puts "Connecting to Logstash: #{config.logstash_host}:#{config.logstash_port}"
|
46
|
-
$logstash = UDPSocket.new
|
47
|
-
$logstash.connect(config.logstash_host, config.logstash_port)
|
48
|
-
else
|
49
|
-
puts "no host:port given"
|
50
|
-
exit 1
|
51
|
-
end
|
52
|
-
$switch_hash = config.switch_hash
|
53
|
-
if config.switch_hash != nil
|
54
|
-
$switchportnames = SNMPwalk.new(config.switch_hash.each_key)
|
55
|
-
end
|
56
|
-
EventMachine::run do
|
57
|
-
EventMachine::open_datagram_socket(bind_ip, bind_port, Collector)
|
58
|
-
end
|
59
|
-
rescue Exception => e
|
60
|
-
puts Time.now
|
61
|
-
puts e.message
|
62
|
-
puts e.backtrace
|
63
|
-
raise "unable to start sflow collector"
|
64
|
-
end
|
65
|
-
end
|
66
|
-
|
67
|
-
end
|
68
|
-
|
69
|
-
|
@@ -1,15 +0,0 @@
|
|
1
|
-
class SflowConfig
|
2
|
-
attr_reader :switch_hash
|
3
|
-
attr_reader :logstash_host
|
4
|
-
attr_reader :logstash_port
|
5
|
-
attr_reader :daemonize
|
6
|
-
|
7
|
-
def initialize
|
8
|
-
config = YAML.load_file("etc/config.yaml")
|
9
|
-
@switch_hash = config['switch']
|
10
|
-
@logstash_host = config['logstash_host']
|
11
|
-
@logstash_port = config['logstash_port']
|
12
|
-
@daemonize = config['daemonize']
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
@@ -1,176 +0,0 @@
|
|
1
|
-
class Header < BinData::Record
|
2
|
-
endian :big
|
3
|
-
uint32 :version
|
4
|
-
uint32 :address_type
|
5
|
-
uint32 :agent_address
|
6
|
-
uint32 :sub_agent_id
|
7
|
-
uint32 :seq_number
|
8
|
-
uint32 :sys_uptime
|
9
|
-
uint32 :num_samples
|
10
|
-
array :flow_samples, :initial_length => :num_samples do
|
11
|
-
uint16 :enterprise_std
|
12
|
-
uint16 :sflow_sample_type
|
13
|
-
uint32 :sample_length
|
14
|
-
string :sample_data, :length => :sample_length
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
class Sflow5sampleheader1 < BinData::Record
|
19
|
-
endian :big
|
20
|
-
uint32 :seq_number
|
21
|
-
uint32 :source_id_type
|
22
|
-
uint32 :sampling_rate
|
23
|
-
uint32 :sample_pool
|
24
|
-
uint32 :dropped_packets
|
25
|
-
uint32 :i_iface_value
|
26
|
-
uint32 :o_iface_value
|
27
|
-
uint32 :num_records
|
28
|
-
array :records, :initial_length => :num_records do
|
29
|
-
uint16 :enterprise
|
30
|
-
uint16 :format
|
31
|
-
uint32 :flow_length
|
32
|
-
string :record_data, :length => :flow_length
|
33
|
-
end
|
34
|
-
|
35
|
-
end
|
36
|
-
|
37
|
-
class Sflow5sampleheader3 < BinData::Record
|
38
|
-
endian :big
|
39
|
-
uint32 :seq_number
|
40
|
-
uint32 :source_id_type
|
41
|
-
uint32 :source_id_index
|
42
|
-
uint32 :sampling_rate
|
43
|
-
uint32 :sample_pool
|
44
|
-
uint32 :dropped_packets
|
45
|
-
uint32 :i_iface_format
|
46
|
-
uint32 :i_iface_value
|
47
|
-
uint32 :o_iface_format
|
48
|
-
uint32 :o_iface_value
|
49
|
-
uint32 :num_records
|
50
|
-
array :records, :initial_length => :num_records do
|
51
|
-
uint16 :enterprise
|
52
|
-
uint16 :format
|
53
|
-
uint32 :flow_length
|
54
|
-
string :record_data, :length => :flow_length
|
55
|
-
end
|
56
|
-
|
57
|
-
end
|
58
|
-
|
59
|
-
|
60
|
-
class Sflow5counterheader4 < BinData::Record
|
61
|
-
endian :big
|
62
|
-
uint32 :seq_number
|
63
|
-
uint32 :source_id_type
|
64
|
-
uint32 :source_id_index
|
65
|
-
uint32 :num_records
|
66
|
-
array :records, :initial_length => :num_records do
|
67
|
-
uint16 :enterprise
|
68
|
-
uint16 :format
|
69
|
-
uint32 :record_length
|
70
|
-
string :record_data, :length => :record_length
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
class Sflow5counterheader2 < BinData::Record
|
75
|
-
endian :big
|
76
|
-
uint32 :seq_number
|
77
|
-
uint32 :source_id_type
|
78
|
-
uint32 :num_records
|
79
|
-
array :records, :initial_length => :num_records do
|
80
|
-
uint16 :enterprise
|
81
|
-
uint16 :format
|
82
|
-
uint32 :record_length
|
83
|
-
string :record_data, :length => :record_length
|
84
|
-
end
|
85
|
-
end
|
86
|
-
|
87
|
-
|
88
|
-
class Sflow5rawpacket < BinData::Record
|
89
|
-
endian :big
|
90
|
-
uint32 :header_protocol
|
91
|
-
uint32 :frame_length
|
92
|
-
uint32 :payload
|
93
|
-
uint32 :xy
|
94
|
-
array :rawpacket_data, :read_until => :eof do
|
95
|
-
string :data, :length => 1
|
96
|
-
end
|
97
|
-
end
|
98
|
-
|
99
|
-
class Sflow5extswitch < BinData::Record
|
100
|
-
endian :big
|
101
|
-
uint32 :src_vlan
|
102
|
-
uint32 :src_priority
|
103
|
-
uint32 :dst_vlan
|
104
|
-
uint32 :dst_priority
|
105
|
-
end
|
106
|
-
|
107
|
-
class Sflow5genericcounter < BinData::Record
|
108
|
-
endian :big
|
109
|
-
uint32 :int_index
|
110
|
-
uint32 :int_type
|
111
|
-
uint64 :int_speed
|
112
|
-
uint32 :int_direction
|
113
|
-
uint16 :int_admin_status
|
114
|
-
uint16 :int_oper_status
|
115
|
-
uint64 :input_octets
|
116
|
-
uint32 :input_packets
|
117
|
-
uint32 :input_packets_multi
|
118
|
-
uint32 :input_packets_broad
|
119
|
-
uint32 :input_packets_discard
|
120
|
-
uint32 :input_packets_error
|
121
|
-
uint32 :unknown_proto
|
122
|
-
uint64 :output_octets
|
123
|
-
uint32 :output_packets
|
124
|
-
uint32 :output_packets_multi
|
125
|
-
uint32 :output_packets_broad
|
126
|
-
uint32 :output_packets_discard
|
127
|
-
uint32 :output_packets_error
|
128
|
-
uint32 :prom_mode
|
129
|
-
end
|
130
|
-
|
131
|
-
class Sflow5ethcounter < BinData::Record
|
132
|
-
endian :big
|
133
|
-
uint32 :alignment_errors
|
134
|
-
uint32 :fcs_errors
|
135
|
-
uint32 :single_collision_frames
|
136
|
-
uint32 :multi_collision_frames
|
137
|
-
uint32 :sqe_test_errors
|
138
|
-
uint32 :deffered_transmission
|
139
|
-
uint32 :late_collision
|
140
|
-
uint32 :excessive_collision
|
141
|
-
uint32 :internal_mac_transmit_errors
|
142
|
-
uint32 :carrier_sense_errors
|
143
|
-
uint32 :frame_too_long
|
144
|
-
uint32 :internal_mac_receive_errors
|
145
|
-
uint32 :symbol_errors
|
146
|
-
end
|
147
|
-
|
148
|
-
class Sflow5rawpacketheaderEthernet < BinData::Record
|
149
|
-
endian :big
|
150
|
-
string :eth_src, :length => 6
|
151
|
-
string :eth_dst, :length => 6
|
152
|
-
uint16 :eth_type
|
153
|
-
array :ethernetdata, :read_until => :eof do
|
154
|
-
string :data, :length => 1
|
155
|
-
end
|
156
|
-
end
|
157
|
-
|
158
|
-
class Sflow5rawpacketdata < BinData::Record
|
159
|
-
endian :big
|
160
|
-
string :eth, :length => 14
|
161
|
-
string :vlan_tag, :length => 2
|
162
|
-
string :vlan_tag_p, :length => 2
|
163
|
-
string :vlana, :length => 2
|
164
|
-
string :vlanb, :length => 2
|
165
|
-
string :ip_packet, :length => 40
|
166
|
-
end
|
167
|
-
|
168
|
-
class Sflow5rawpacketdataVLAN < BinData::Record
|
169
|
-
endian :big
|
170
|
-
uint16 :prio
|
171
|
-
uint16 :type
|
172
|
-
array :vlandata, :read_until => :eof do
|
173
|
-
string :data, :length => 1
|
174
|
-
end
|
175
|
-
end
|
176
|
-
|