fluent-plugin-sflow 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +18 -4
  3. data/README.md +254 -13
  4. data/Rakefile +10 -7
  5. data/example/fluentd.conf +4 -2
  6. data/ext/sflowtool/extconf.rb +27 -0
  7. data/ext/sflowtool/fluent-plugin-sflow.c +52 -0
  8. data/ext/sflowtool/sflow.h +1821 -0
  9. data/ext/sflowtool/sflow_v2v4.h +436 -0
  10. data/ext/sflowtool/sflowtool.c +4801 -0
  11. data/fluent-plugin-sflow.gemspec +11 -25
  12. data/lib/fluent/plugin/in_sflow.rb +66 -23
  13. data/lib/fluent/plugin/parser_sflow.rb +24 -0
  14. data/lib/sflowtool.rb +17 -0
  15. metadata +35 -84
  16. data/.gitmodules +0 -3
  17. data/LICENSE.txt +0 -21
  18. data/lib/sflow/Gemfile +0 -8
  19. data/lib/sflow/Gemfile.lock +0 -38
  20. data/lib/sflow/LICENSE.txt +0 -22
  21. data/lib/sflow/README.md +0 -67
  22. data/lib/sflow/Rakefile +0 -12
  23. data/lib/sflow/bin/bundler +0 -16
  24. data/lib/sflow/bin/rake +0 -16
  25. data/lib/sflow/bin/sflow.rb +0 -7
  26. data/lib/sflow/etc/config.yaml +0 -10
  27. data/lib/sflow/lib/sflow.rb +0 -10
  28. data/lib/sflow/lib/sflow/collector.rb +0 -69
  29. data/lib/sflow/lib/sflow/config.rb +0 -15
  30. data/lib/sflow/lib/sflow/models/binary_models.rb +0 -176
  31. data/lib/sflow/lib/sflow/models/ipv4header.rb +0 -69
  32. data/lib/sflow/lib/sflow/models/protocol.rb +0 -47
  33. data/lib/sflow/lib/sflow/models/tcpheader.rb +0 -82
  34. data/lib/sflow/lib/sflow/models/udpheader.rb +0 -36
  35. data/lib/sflow/lib/sflow/parsers/parsers.rb +0 -68
  36. data/lib/sflow/lib/sflow/snmp/iface_names.rb +0 -40
  37. data/lib/sflow/lib/sflow/storage/storage.rb +0 -34
  38. data/lib/sflow/lib/sflow/version.rb +0 -3
  39. data/lib/sflow/misc/kibana-schema.json +0 -1364
  40. data/lib/sflow/misc/screen1.png +0 -0
  41. data/lib/sflow/sflow.gemspec +0 -23
  42. data/lib/sflow/test/lib/sflow/version_test.rb +0 -8
  43. data/lib/sflow/test/test_helper.rb +0 -4
@@ -1,38 +0,0 @@
1
- PATH
2
- remote: .
3
- specs:
4
- sflow (0.0.1)
5
-
6
- GEM
7
- remote: https://rubygems.org/
8
- specs:
9
- bindata (1.8.1)
10
- elasticsearch (1.0.1)
11
- elasticsearch-api (= 1.0.1)
12
- elasticsearch-transport (= 1.0.1)
13
- elasticsearch-api (1.0.1)
14
- multi_json
15
- elasticsearch-transport (1.0.1)
16
- faraday
17
- multi_json
18
- eventmachine (1.0.3)
19
- faraday (0.8.8)
20
- multipart-post (~> 1.2.0)
21
- json (1.8.1)
22
- minitest (5.3.4)
23
- multi_json (1.7.9)
24
- multipart-post (1.2.0)
25
- rake (10.2.0)
26
-
27
- PLATFORMS
28
- ruby
29
-
30
- DEPENDENCIES
31
- bindata
32
- bundler (~> 1.5)
33
- elasticsearch
34
- eventmachine
35
- json
36
- minitest
37
- rake
38
- sflow!
@@ -1,22 +0,0 @@
1
- Copyright (c) 2014 Sebastian Saemann <ssaemann@netways.de>
2
-
3
- MIT License
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining
6
- a copy of this software and associated documentation files (the
7
- "Software"), to deal in the Software without restriction, including
8
- without limitation the rights to use, copy, modify, merge, publish,
9
- distribute, sublicense, and/or sell copies of the Software, and to
10
- permit persons to whom the Software is furnished to do so, subject to
11
- the following conditions:
12
-
13
- The above copyright notice and this permission notice shall be
14
- included in all copies or substantial portions of the Software.
15
-
16
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
- NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
- LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
- OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/lib/sflow/README.md DELETED
@@ -1,67 +0,0 @@
1
- # Sflow
2
-
3
- Tiny sflow collector and parser script based on eventmachine. It listens for sflow v5 samples, parses them and sends it to logstash.
4
-
5
- ## Installation
6
-
7
- Clone this repository
8
-
9
- $ git clone http://github.com/netways/sflow
10
-
11
- Change directory
12
-
13
- $ cd sflow
14
-
15
- Install dependencies using bundler
16
-
17
- $ bundle install
18
-
19
- Configure your logstash endpoint
20
-
21
- $ vi ./etc/config.yaml
22
-
23
- And then execute:
24
-
25
- $ bundle exec ./bin/sflow.rb
26
-
27
- ## Logstash Configuration
28
-
29
- A complete logstash installation is a prerequisite.
30
-
31
- For getting the parsed sflow-packets as JSON via UDP into logstash you have to configure a input, filter and a output accordingly:
32
-
33
- input {
34
- udp {
35
- port => 6543
36
- type => "sflow"
37
- codec => 'json'
38
- }
39
- }
40
-
41
- filter {
42
- json {
43
- source => "message"
44
- type => "json"
45
- }
46
- }
47
-
48
- output {
49
- elasticsearch_http {
50
- workers => 8
51
- host => "elasticsearch.host"
52
- }
53
- }
54
-
55
- ## Kibana
56
-
57
- You can create your very own kibana dashboard for viewing the information and graphs you are interested in. For a quick start you'll find a dashboard in the misc folder, which can be imported via the kibana webinterface.
58
-
59
- ![Alt text](misc/screen1.png?raw=true "Demo screen")
60
-
61
- ## Contributing
62
-
63
- 1. Fork it ( http://github.com/netways/sflow/fork )
64
- 2. Create your feature branch (`git checkout -b my-new-feature`)
65
- 3. Commit your changes (`git commit -am 'Add some feature'`)
66
- 4. Push to the branch (`git push origin my-new-feature`)
67
- 5. Create new Pull Request
data/lib/sflow/Rakefile DELETED
@@ -1,12 +0,0 @@
1
- #!/usr/bin/env rake
2
- require "bundler/gem_tasks"
3
-
4
- require 'rake/testtask'
5
-
6
- Rake::TestTask.new do |t|
7
- t.libs << 'lib/sflow'
8
- t.test_files = FileList['test/lib/sflow/*_test.rb']
9
- t.verbose = true
10
- end
11
-
12
- task :default => :test
@@ -1,16 +0,0 @@
1
- #!/usr/bin/env ruby1.9.1
2
- #
3
- # This file was generated by Bundler.
4
- #
5
- # The application 'bundler' is installed as part of a gem, and
6
- # this file is here to facilitate running it.
7
- #
8
-
9
- require 'pathname'
10
- ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
11
- Pathname.new(__FILE__).realpath)
12
-
13
- require 'rubygems'
14
- require 'bundler/setup'
15
-
16
- load Gem.bin_path('bundler', 'bundler')
data/lib/sflow/bin/rake DELETED
@@ -1,16 +0,0 @@
1
- #!/usr/bin/env ruby1.9.1
2
- #
3
- # This file was generated by Bundler.
4
- #
5
- # The application 'rake' is installed as part of a gem, and
6
- # this file is here to facilitate running it.
7
- #
8
-
9
- require 'pathname'
10
- ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
11
- Pathname.new(__FILE__).realpath)
12
-
13
- require 'rubygems'
14
- require 'bundler/setup'
15
-
16
- load Gem.bin_path('rake', 'rake')
@@ -1,7 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- $:.unshift File.expand_path(File.join(File.dirname(__FILE__), '..','lib'))
4
-
5
- require 'sflow'
6
-
7
- SflowCollector.start_collector('0.0.0.0',6343)
@@ -1,10 +0,0 @@
1
- daemonize: true
2
-
3
- switch:
4
- 1.2.3.4: "myswitch_hostname"
5
- 1.2.3.5: "my2ndswitch_hostname"
6
-
7
- logstash_host: "logstash.host"
8
- logstash_port: 6543
9
-
10
-
@@ -1,10 +0,0 @@
1
- require 'bindata'
2
- require 'eventmachine'
3
- require 'yaml'
4
-
5
- dir = File.expand_path(File.join(File.dirname(__FILE__), 'sflow'))
6
- ['config','models/ipv4header', 'models/tcpheader', 'models/udpheader', 'models/protocol', 'models/binary_models','parsers/parsers','storage/storage', 'collector','snmp/iface_names'].each do |req|
7
- require File.join(dir, req)
8
- end
9
-
10
- Process.daemon(true) if $daemonize == true
@@ -1,69 +0,0 @@
1
- class SflowCollector
2
- module Collector
3
- Thread.abort_on_exception=true
4
- require 'socket'
5
- def post_init
6
- puts "Server listening."
7
- end
8
-
9
- def receive_data(data)
10
- operation = proc do
11
- begin
12
- if data != nil
13
- sflow = SflowParser.parse_packet(data)
14
- end
15
- rescue Exception => e
16
- puts Time.now
17
- puts sflow.inspect
18
- puts e.message
19
- puts e.backtrace
20
- end
21
- end
22
-
23
- callback = proc do |sflow|
24
- begin
25
- if sflow != nil
26
- SflowStorage.send_udpjson(sflow)
27
- end
28
- rescue Exception => e
29
- puts Time.now
30
- puts sflow.inspect if sflow != nil
31
- puts e.message
32
- puts e.backtrace
33
- end
34
- end
35
-
36
- EM.defer(operation,callback)
37
-
38
- end
39
- end
40
-
41
- def self.start_collector(bind_ip = '0.0.0.0', bind_port = 6343)
42
- begin
43
- config = SflowConfig.new
44
- if config.logstash_host and config.logstash_port
45
- puts "Connecting to Logstash: #{config.logstash_host}:#{config.logstash_port}"
46
- $logstash = UDPSocket.new
47
- $logstash.connect(config.logstash_host, config.logstash_port)
48
- else
49
- puts "no host:port given"
50
- exit 1
51
- end
52
- $switch_hash = config.switch_hash
53
- if config.switch_hash != nil
54
- $switchportnames = SNMPwalk.new(config.switch_hash.each_key)
55
- end
56
- EventMachine::run do
57
- EventMachine::open_datagram_socket(bind_ip, bind_port, Collector)
58
- end
59
- rescue Exception => e
60
- puts Time.now
61
- puts e.message
62
- puts e.backtrace
63
- raise "unable to start sflow collector"
64
- end
65
- end
66
-
67
- end
68
-
69
-
@@ -1,15 +0,0 @@
1
- class SflowConfig
2
- attr_reader :switch_hash
3
- attr_reader :logstash_host
4
- attr_reader :logstash_port
5
- attr_reader :daemonize
6
-
7
- def initialize
8
- config = YAML.load_file("etc/config.yaml")
9
- @switch_hash = config['switch']
10
- @logstash_host = config['logstash_host']
11
- @logstash_port = config['logstash_port']
12
- @daemonize = config['daemonize']
13
- end
14
- end
15
-
@@ -1,176 +0,0 @@
1
- class Header < BinData::Record
2
- endian :big
3
- uint32 :version
4
- uint32 :address_type
5
- uint32 :agent_address
6
- uint32 :sub_agent_id
7
- uint32 :seq_number
8
- uint32 :sys_uptime
9
- uint32 :num_samples
10
- array :flow_samples, :initial_length => :num_samples do
11
- uint16 :enterprise_std
12
- uint16 :sflow_sample_type
13
- uint32 :sample_length
14
- string :sample_data, :length => :sample_length
15
- end
16
- end
17
-
18
- class Sflow5sampleheader1 < BinData::Record
19
- endian :big
20
- uint32 :seq_number
21
- uint32 :source_id_type
22
- uint32 :sampling_rate
23
- uint32 :sample_pool
24
- uint32 :dropped_packets
25
- uint32 :i_iface_value
26
- uint32 :o_iface_value
27
- uint32 :num_records
28
- array :records, :initial_length => :num_records do
29
- uint16 :enterprise
30
- uint16 :format
31
- uint32 :flow_length
32
- string :record_data, :length => :flow_length
33
- end
34
-
35
- end
36
-
37
- class Sflow5sampleheader3 < BinData::Record
38
- endian :big
39
- uint32 :seq_number
40
- uint32 :source_id_type
41
- uint32 :source_id_index
42
- uint32 :sampling_rate
43
- uint32 :sample_pool
44
- uint32 :dropped_packets
45
- uint32 :i_iface_format
46
- uint32 :i_iface_value
47
- uint32 :o_iface_format
48
- uint32 :o_iface_value
49
- uint32 :num_records
50
- array :records, :initial_length => :num_records do
51
- uint16 :enterprise
52
- uint16 :format
53
- uint32 :flow_length
54
- string :record_data, :length => :flow_length
55
- end
56
-
57
- end
58
-
59
-
60
- class Sflow5counterheader4 < BinData::Record
61
- endian :big
62
- uint32 :seq_number
63
- uint32 :source_id_type
64
- uint32 :source_id_index
65
- uint32 :num_records
66
- array :records, :initial_length => :num_records do
67
- uint16 :enterprise
68
- uint16 :format
69
- uint32 :record_length
70
- string :record_data, :length => :record_length
71
- end
72
- end
73
-
74
- class Sflow5counterheader2 < BinData::Record
75
- endian :big
76
- uint32 :seq_number
77
- uint32 :source_id_type
78
- uint32 :num_records
79
- array :records, :initial_length => :num_records do
80
- uint16 :enterprise
81
- uint16 :format
82
- uint32 :record_length
83
- string :record_data, :length => :record_length
84
- end
85
- end
86
-
87
-
88
- class Sflow5rawpacket < BinData::Record
89
- endian :big
90
- uint32 :header_protocol
91
- uint32 :frame_length
92
- uint32 :payload
93
- uint32 :xy
94
- array :rawpacket_data, :read_until => :eof do
95
- string :data, :length => 1
96
- end
97
- end
98
-
99
- class Sflow5extswitch < BinData::Record
100
- endian :big
101
- uint32 :src_vlan
102
- uint32 :src_priority
103
- uint32 :dst_vlan
104
- uint32 :dst_priority
105
- end
106
-
107
- class Sflow5genericcounter < BinData::Record
108
- endian :big
109
- uint32 :int_index
110
- uint32 :int_type
111
- uint64 :int_speed
112
- uint32 :int_direction
113
- uint16 :int_admin_status
114
- uint16 :int_oper_status
115
- uint64 :input_octets
116
- uint32 :input_packets
117
- uint32 :input_packets_multi
118
- uint32 :input_packets_broad
119
- uint32 :input_packets_discard
120
- uint32 :input_packets_error
121
- uint32 :unknown_proto
122
- uint64 :output_octets
123
- uint32 :output_packets
124
- uint32 :output_packets_multi
125
- uint32 :output_packets_broad
126
- uint32 :output_packets_discard
127
- uint32 :output_packets_error
128
- uint32 :prom_mode
129
- end
130
-
131
- class Sflow5ethcounter < BinData::Record
132
- endian :big
133
- uint32 :alignment_errors
134
- uint32 :fcs_errors
135
- uint32 :single_collision_frames
136
- uint32 :multi_collision_frames
137
- uint32 :sqe_test_errors
138
- uint32 :deffered_transmission
139
- uint32 :late_collision
140
- uint32 :excessive_collision
141
- uint32 :internal_mac_transmit_errors
142
- uint32 :carrier_sense_errors
143
- uint32 :frame_too_long
144
- uint32 :internal_mac_receive_errors
145
- uint32 :symbol_errors
146
- end
147
-
148
- class Sflow5rawpacketheaderEthernet < BinData::Record
149
- endian :big
150
- string :eth_src, :length => 6
151
- string :eth_dst, :length => 6
152
- uint16 :eth_type
153
- array :ethernetdata, :read_until => :eof do
154
- string :data, :length => 1
155
- end
156
- end
157
-
158
- class Sflow5rawpacketdata < BinData::Record
159
- endian :big
160
- string :eth, :length => 14
161
- string :vlan_tag, :length => 2
162
- string :vlan_tag_p, :length => 2
163
- string :vlana, :length => 2
164
- string :vlanb, :length => 2
165
- string :ip_packet, :length => 40
166
- end
167
-
168
- class Sflow5rawpacketdataVLAN < BinData::Record
169
- endian :big
170
- uint16 :prio
171
- uint16 :type
172
- array :vlandata, :read_until => :eof do
173
- string :data, :length => 1
174
- end
175
- end
176
-