fluent-plugin-netflow 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.travis.yml +19 -0
- data/Gemfile +3 -0
- data/README.md +28 -0
- data/Rakefile +14 -0
- data/VERSION +1 -0
- data/fluent-plugin-netflow.gemspec +23 -0
- data/lib/fluent/plugin/in_netflow.rb +115 -0
- data/lib/fluent/plugin/netflow.yaml +215 -0
- data/lib/fluent/plugin/parser_netflow.rb +448 -0
- metadata +94 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: c83a5f4fb673a2350abaab3f8d33ca43229399e5
|
4
|
+
data.tar.gz: 8a24c6097b65b0a7eac8391cc9a54444c4a49a17
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: ce6510d36beed459299342e64ae6336b004d9c924886336edf57c214ff29405dc96d9764976147d383d192e70bfed7b935e6156f3491678c95fe306628f3fe2b
|
7
|
+
data.tar.gz: ce0a4789512e66527d777ae8b6b37958b6d94221874ee91651f8764019095efd308f5c70b44e5f9d980a9cc6c0d643a1e7107eea43997a632765d87978a02e66
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# Netflow plugin for Fluentd
|
2
|
+
|
3
|
+
Accept Netflow logs.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Use RubyGems:
|
8
|
+
|
9
|
+
gem install fluent-plugin-netflow
|
10
|
+
|
11
|
+
## Configuration
|
12
|
+
|
13
|
+
<source>
|
14
|
+
type netflow
|
15
|
+
tag netflow.event
|
16
|
+
|
17
|
+
# optional parameters
|
18
|
+
bind 127.0.0.1
|
19
|
+
port 5140
|
20
|
+
|
21
|
+
# optional parser parameters
|
22
|
+
cache_ttl 6000
|
23
|
+
versions [5, 9]
|
24
|
+
</match>
|
25
|
+
|
26
|
+
## TODO
|
27
|
+
|
28
|
+
* Release as rubygem
|
data/Rakefile
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
|
2
|
+
require 'bundler'
|
3
|
+
Bundler::GemHelper.install_tasks
|
4
|
+
|
5
|
+
require 'rake/testtask'
|
6
|
+
|
7
|
+
Rake::TestTask.new(:test) do |test|
|
8
|
+
test.libs << 'lib' << 'test'
|
9
|
+
test.test_files = FileList['test/*.rb']
|
10
|
+
test.verbose = true
|
11
|
+
end
|
12
|
+
|
13
|
+
task :default => [:build]
|
14
|
+
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.0.1
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.name = "fluent-plugin-netflow"
|
6
|
+
gem.description = "Netflow plugin for Fluentd"
|
7
|
+
gem.homepage = "https://github.com/repeatedly/fluent-plugin-netflow"
|
8
|
+
gem.summary = gem.description
|
9
|
+
gem.version = File.read("VERSION").strip
|
10
|
+
gem.authors = ["Masahiro Nakagawa"]
|
11
|
+
gem.email = "repeatedly@gmail.com"
|
12
|
+
gem.has_rdoc = false
|
13
|
+
#gem.platform = Gem::Platform::RUBY
|
14
|
+
gem.license = 'Apache License (2.0)'
|
15
|
+
gem.files = `git ls-files`.split("\n")
|
16
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
gem.require_paths = ['lib']
|
19
|
+
|
20
|
+
gem.add_dependency "fluentd", "~> 0.10.17"
|
21
|
+
gem.add_dependency "bindata", "~> 2.1"
|
22
|
+
gem.add_development_dependency "rake", ">= 0.9.2"
|
23
|
+
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
#
|
2
|
+
# Fluent
|
3
|
+
#
|
4
|
+
# Copyright (C) 2014 Masahiro Nakagawa
|
5
|
+
#
|
6
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
7
|
+
# you may not use this file except in compliance with the License.
|
8
|
+
# You may obtain a copy of the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
14
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
15
|
+
# See the License for the specific language governing permissions and
|
16
|
+
# limitations under the License.
|
17
|
+
#
|
18
|
+
module Fluent
|
19
|
+
class NetflowInput < Input
|
20
|
+
Plugin.register_input('netflow', self)
|
21
|
+
|
22
|
+
def initialize
|
23
|
+
super
|
24
|
+
require 'cool.io'
|
25
|
+
require 'fluent/plugin/socket_util'
|
26
|
+
require 'fluent/plugin/parser_netflow'
|
27
|
+
end
|
28
|
+
|
29
|
+
config_param :port, :integer, :default => 5140
|
30
|
+
config_param :bind, :string, :default => '0.0.0.0'
|
31
|
+
config_param :tag, :string
|
32
|
+
config_param :protocol_type, :default => :udp do |val|
|
33
|
+
case val.downcase
|
34
|
+
when 'udp'
|
35
|
+
:udp
|
36
|
+
else
|
37
|
+
raise ConfigError, "syslog input protocol type should be 'tcp' or 'udp'"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def configure(conf)
|
42
|
+
super
|
43
|
+
|
44
|
+
@parser = TextParser::NetflowParser.new
|
45
|
+
@parser.configure(conf)
|
46
|
+
end
|
47
|
+
|
48
|
+
def start
|
49
|
+
@loop = Coolio::Loop.new
|
50
|
+
@handler = listen(method(:receive_data))
|
51
|
+
@loop.attach(@handler)
|
52
|
+
|
53
|
+
@thread = Thread.new(&method(:run))
|
54
|
+
end
|
55
|
+
|
56
|
+
def shutdown
|
57
|
+
@loop.watchers.each { |w| w.detach }
|
58
|
+
@loop.stop
|
59
|
+
@handler.close
|
60
|
+
@thread.join
|
61
|
+
end
|
62
|
+
|
63
|
+
def run
|
64
|
+
@loop.run
|
65
|
+
rescue
|
66
|
+
log.error "unexpected error", :error=>$!.to_s
|
67
|
+
log.error_backtrace
|
68
|
+
end
|
69
|
+
|
70
|
+
protected
|
71
|
+
|
72
|
+
def receive_data(host, data)
|
73
|
+
@parser.call(data) { |time, record|
|
74
|
+
unless time && record
|
75
|
+
log.warn "pattern not match: #{text.inspect}"
|
76
|
+
return
|
77
|
+
end
|
78
|
+
|
79
|
+
record['host'] = host
|
80
|
+
Engine.emit(tag, time, record)
|
81
|
+
}
|
82
|
+
rescue
|
83
|
+
log.warn data.dump, :error => $!.to_s
|
84
|
+
log.debug_backtrace
|
85
|
+
end
|
86
|
+
|
87
|
+
private
|
88
|
+
|
89
|
+
def listen(callback)
|
90
|
+
log.debug "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
|
91
|
+
if @protocol_type == :udp
|
92
|
+
@usock = SocketUtil.create_udp_socket(@bind)
|
93
|
+
@usock.bind(@bind, @port)
|
94
|
+
UdpHandler.new(@usock, callback)
|
95
|
+
else
|
96
|
+
Coolio::TCPServer.new(@bind, @port, TcpHandler, log, callback)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
class UdpHandler < Coolio::IO
|
101
|
+
def initialize(io, callback)
|
102
|
+
super(io)
|
103
|
+
@io = io
|
104
|
+
@callback = callback
|
105
|
+
end
|
106
|
+
|
107
|
+
def on_readable
|
108
|
+
msg, addr = @io.recvfrom_nonblock(4096)
|
109
|
+
@callback.call(addr[3], msg)
|
110
|
+
rescue
|
111
|
+
# TODO log?
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -0,0 +1,215 @@
|
|
1
|
+
---
|
2
|
+
1:
|
3
|
+
- 4
|
4
|
+
- :in_bytes
|
5
|
+
2:
|
6
|
+
- 4
|
7
|
+
- :in_pkts
|
8
|
+
3:
|
9
|
+
- 4
|
10
|
+
- :flows
|
11
|
+
4:
|
12
|
+
- :uint8
|
13
|
+
- :protocol
|
14
|
+
5:
|
15
|
+
- :uint8
|
16
|
+
- :src_tos
|
17
|
+
6:
|
18
|
+
- :uint8
|
19
|
+
- :tcp_flags
|
20
|
+
7:
|
21
|
+
- :uint16
|
22
|
+
- :l4_src_port
|
23
|
+
8:
|
24
|
+
- :ip4_addr
|
25
|
+
- :ipv4_src_addr
|
26
|
+
9:
|
27
|
+
- :uint8
|
28
|
+
- :src_mask
|
29
|
+
10:
|
30
|
+
- 2
|
31
|
+
- :input_snmp
|
32
|
+
11:
|
33
|
+
- :uint16
|
34
|
+
- :l4_dst_port
|
35
|
+
12:
|
36
|
+
- :ip4_addr
|
37
|
+
- :ipv4_dst_addr
|
38
|
+
13:
|
39
|
+
- :uint8
|
40
|
+
- :dst_mask
|
41
|
+
14:
|
42
|
+
- 2
|
43
|
+
- :output_snmp
|
44
|
+
15:
|
45
|
+
- :ip4_addr
|
46
|
+
- :ipv4_next_hop
|
47
|
+
16:
|
48
|
+
- 2
|
49
|
+
- :src_as
|
50
|
+
17:
|
51
|
+
- 2
|
52
|
+
- :dst_as
|
53
|
+
18:
|
54
|
+
- :ip4_addr
|
55
|
+
- :bgp_ipv4_next_hop
|
56
|
+
19:
|
57
|
+
- 4
|
58
|
+
- :mul_dst_pkts
|
59
|
+
20:
|
60
|
+
- 4
|
61
|
+
- :mul_dst_bytes
|
62
|
+
21:
|
63
|
+
- :uint32
|
64
|
+
- :last_switched
|
65
|
+
22:
|
66
|
+
- :uint32
|
67
|
+
- :first_switched
|
68
|
+
23:
|
69
|
+
- 4
|
70
|
+
- :out_bytes
|
71
|
+
24:
|
72
|
+
- 4
|
73
|
+
- :out_pkts
|
74
|
+
25:
|
75
|
+
- :uint16
|
76
|
+
- :min_pkt_length
|
77
|
+
26:
|
78
|
+
- :uint16
|
79
|
+
- :max_pkt_length
|
80
|
+
27:
|
81
|
+
- :ip6_addr
|
82
|
+
- :ipv6_src_addr
|
83
|
+
28:
|
84
|
+
- :ip6_addr
|
85
|
+
- :ipv6_dst_addr
|
86
|
+
29:
|
87
|
+
- :uint8
|
88
|
+
- :ipv6_src_mask
|
89
|
+
30:
|
90
|
+
- :uint8
|
91
|
+
- :ipv6_dst_mask
|
92
|
+
31:
|
93
|
+
- :uint24
|
94
|
+
- :ipv6_flow_label
|
95
|
+
32:
|
96
|
+
- :uint16
|
97
|
+
- :icmp_type
|
98
|
+
33:
|
99
|
+
- :uint8
|
100
|
+
- :mul_igmp_type
|
101
|
+
34:
|
102
|
+
- :uint32
|
103
|
+
- :sampling_interval
|
104
|
+
35:
|
105
|
+
- :uint8
|
106
|
+
- :sampling_algorithm
|
107
|
+
36:
|
108
|
+
- :uint16
|
109
|
+
- :flow_active_timeout
|
110
|
+
37:
|
111
|
+
- :uint16
|
112
|
+
- :flow_inactive_timeout
|
113
|
+
38:
|
114
|
+
- :uint8
|
115
|
+
- :engine_type
|
116
|
+
39:
|
117
|
+
- :uint8
|
118
|
+
- :engine_id
|
119
|
+
40:
|
120
|
+
- 4
|
121
|
+
- :total_bytes_exp
|
122
|
+
41:
|
123
|
+
- 4
|
124
|
+
- :total_pkts_exp
|
125
|
+
42:
|
126
|
+
- 4
|
127
|
+
- :total_flows_exp
|
128
|
+
43:
|
129
|
+
- :skip
|
130
|
+
44:
|
131
|
+
- :ip4_addr
|
132
|
+
- :ipv4_src_prefix
|
133
|
+
45:
|
134
|
+
- :ip4_addr
|
135
|
+
- :ipv4_dst_prefix
|
136
|
+
46:
|
137
|
+
- :uint8
|
138
|
+
- :mpls_top_label_type
|
139
|
+
47:
|
140
|
+
- :uint32
|
141
|
+
- :mpls_top_label_ip_addr
|
142
|
+
48:
|
143
|
+
- 4
|
144
|
+
- :flow_sampler_id
|
145
|
+
49:
|
146
|
+
- :uint8
|
147
|
+
- :flow_sampler_mode
|
148
|
+
50:
|
149
|
+
- :uint32
|
150
|
+
- :flow_sampler_random_interval
|
151
|
+
51:
|
152
|
+
- :skip
|
153
|
+
52:
|
154
|
+
- :uint8
|
155
|
+
- :min_ttl
|
156
|
+
53:
|
157
|
+
- :uint8
|
158
|
+
- :max_ttl
|
159
|
+
54:
|
160
|
+
- :uint16
|
161
|
+
- :ipv4_ident
|
162
|
+
55:
|
163
|
+
- :uint8
|
164
|
+
- :dst_tos
|
165
|
+
56:
|
166
|
+
- :mac_addr
|
167
|
+
- :in_src_max
|
168
|
+
57:
|
169
|
+
- :mac_addr
|
170
|
+
- :out_dst_max
|
171
|
+
58:
|
172
|
+
- :uint16
|
173
|
+
- :src_vlan
|
174
|
+
59:
|
175
|
+
- :uint16
|
176
|
+
- :dst_vlan
|
177
|
+
60:
|
178
|
+
- :uint8
|
179
|
+
- :ip_protocol_version
|
180
|
+
61:
|
181
|
+
- :uint8
|
182
|
+
- :direction
|
183
|
+
62:
|
184
|
+
- :ip6_addr
|
185
|
+
- :ipv6_next_hop
|
186
|
+
63:
|
187
|
+
- :ip6_addr
|
188
|
+
- :bgp_ipv6_next_hop
|
189
|
+
64:
|
190
|
+
- :uint32
|
191
|
+
- :ipv6_option_headers
|
192
|
+
64:
|
193
|
+
- :skip
|
194
|
+
65:
|
195
|
+
- :skip
|
196
|
+
66:
|
197
|
+
- :skip
|
198
|
+
67:
|
199
|
+
- :skip
|
200
|
+
68:
|
201
|
+
- :skip
|
202
|
+
69:
|
203
|
+
- :skip
|
204
|
+
80:
|
205
|
+
- :mac_addr
|
206
|
+
- :in_dst_mac
|
207
|
+
81:
|
208
|
+
- :mac_addr
|
209
|
+
- :out_src_mac
|
210
|
+
82:
|
211
|
+
- :string
|
212
|
+
- :if_name
|
213
|
+
83:
|
214
|
+
- :string
|
215
|
+
- :if_desc
|
@@ -0,0 +1,448 @@
|
|
1
|
+
require "bindata"
|
2
|
+
require "ipaddr"
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
require 'fluent/parser'
|
6
|
+
|
7
|
+
module Fluent
|
8
|
+
class TextParser
|
9
|
+
# port from logstash's netflow parser
|
10
|
+
class NetflowParser
|
11
|
+
include Configurable
|
12
|
+
|
13
|
+
config_param :cache_ttl, :integer, :default => 4000
|
14
|
+
config_param :versions, :default => [5, 9] do |param|
|
15
|
+
if param.is_a?(Array)
|
16
|
+
param
|
17
|
+
else
|
18
|
+
param.split(".").map(&:to_i)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
config_param :definitions, :string, :default => nil
|
22
|
+
|
23
|
+
def configure(conf)
|
24
|
+
super
|
25
|
+
|
26
|
+
@templates = Vash.new()
|
27
|
+
# Path to default Netflow v9 field definitions
|
28
|
+
filename = File.expand_path('../netflow.yaml', __FILE__)
|
29
|
+
|
30
|
+
begin
|
31
|
+
@fields = YAML.load_file(filename)
|
32
|
+
rescue Exception => e
|
33
|
+
raise "Bad syntax in definitions file #{filename}"
|
34
|
+
end
|
35
|
+
|
36
|
+
# Allow the user to augment/override/rename the supported Netflow fields
|
37
|
+
if @definitions
|
38
|
+
raise "definitions file #{@definitions} does not exists" unless File.exist?(@definitions)
|
39
|
+
begin
|
40
|
+
@fields.merge!(YAML.load_file(@definitions))
|
41
|
+
rescue Exception => e
|
42
|
+
raise "Bad syntax in definitions file #{@definitions}"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def call(payload)
|
48
|
+
header = Header.read(payload)
|
49
|
+
unless @versions.include?(header.version)
|
50
|
+
$log.warn "Ignoring Netflow version v#{header.version}"
|
51
|
+
return
|
52
|
+
end
|
53
|
+
|
54
|
+
if header.version == 5
|
55
|
+
flowset = Netflow5PDU.read(payload)
|
56
|
+
elsif header.version == 9
|
57
|
+
flowset = Netflow9PDU.read(payload)
|
58
|
+
else
|
59
|
+
$log.warn "Unsupported Netflow version v#{header.version}"
|
60
|
+
return
|
61
|
+
end
|
62
|
+
|
63
|
+
flowset.records.each do |record|
|
64
|
+
if flowset.version == 5
|
65
|
+
event = {}
|
66
|
+
|
67
|
+
# FIXME Probably not doing this right WRT JRuby?
|
68
|
+
#
|
69
|
+
# The flowset header gives us the UTC epoch seconds along with
|
70
|
+
# residual nanoseconds so we can set @timestamp to that easily
|
71
|
+
time = flowset.unix_sec
|
72
|
+
|
73
|
+
# Copy some of the pertinent fields in the header to the event
|
74
|
+
['version', 'flow_seq_num', 'engine_type', 'engine_id', 'sampling_algorithm', 'sampling_interval', 'flow_records'].each do |f|
|
75
|
+
event[f] = flowset[f]
|
76
|
+
end
|
77
|
+
|
78
|
+
# Create fields in the event from each field in the flow record
|
79
|
+
record.each_pair do |k,v|
|
80
|
+
case k.to_s
|
81
|
+
when /_switched$/
|
82
|
+
# The flow record sets the first and last times to the device
|
83
|
+
# uptime in milliseconds. Given the actual uptime is provided
|
84
|
+
# in the flowset header along with the epoch seconds we can
|
85
|
+
# convert these into absolute times
|
86
|
+
millis = flowset.uptime - v
|
87
|
+
seconds = flowset.unix_sec - (millis / 1000)
|
88
|
+
micros = (flowset.unix_nsec / 1000) - (millis % 1000)
|
89
|
+
if micros < 0
|
90
|
+
seconds--
|
91
|
+
micros += 1000000
|
92
|
+
end
|
93
|
+
|
94
|
+
# FIXME Again, probably doing this wrong WRT JRuby?
|
95
|
+
event[k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
|
96
|
+
else
|
97
|
+
event[k.to_s] = v
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
yield time, event
|
102
|
+
elsif flowset.version == 9
|
103
|
+
case record.flowset_id
|
104
|
+
when 0
|
105
|
+
# Template flowset
|
106
|
+
record.flowset_data.templates.each do |template|
|
107
|
+
catch (:field) do
|
108
|
+
fields = []
|
109
|
+
template.fields.each do |field|
|
110
|
+
entry = netflow_field_for(field.field_type, field.field_length)
|
111
|
+
if !entry
|
112
|
+
throw :field
|
113
|
+
end
|
114
|
+
fields += entry
|
115
|
+
end
|
116
|
+
# We get this far, we have a list of fields
|
117
|
+
#key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
|
118
|
+
key = "#{flowset.source_id}|#{template.template_id}"
|
119
|
+
@templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
|
120
|
+
# Purge any expired templates
|
121
|
+
@templates.cleanup!
|
122
|
+
end
|
123
|
+
end
|
124
|
+
when 1
|
125
|
+
# Options template flowset
|
126
|
+
record.flowset_data.templates.each do |template|
|
127
|
+
catch (:field) do
|
128
|
+
fields = []
|
129
|
+
template.option_fields.each do |field|
|
130
|
+
entry = netflow_field_for(field.field_type, field.field_length)
|
131
|
+
if ! entry
|
132
|
+
throw :field
|
133
|
+
end
|
134
|
+
fields += entry
|
135
|
+
end
|
136
|
+
# We get this far, we have a list of fields
|
137
|
+
#key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
|
138
|
+
key = "#{flowset.source_id}|#{template.template_id}"
|
139
|
+
@templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
|
140
|
+
# Purge any expired templates
|
141
|
+
@templates.cleanup!
|
142
|
+
end
|
143
|
+
end
|
144
|
+
when 256..65535
|
145
|
+
# Data flowset
|
146
|
+
#key = "#{flowset.source_id}|#{event["source"]}|#{record.flowset_id}"
|
147
|
+
key = "#{flowset.source_id}|#{record.flowset_id}"
|
148
|
+
template = @templates[key]
|
149
|
+
if ! template
|
150
|
+
#$log.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
|
151
|
+
$log.warn("No matching template for flow id #{record.flowset_id}")
|
152
|
+
next
|
153
|
+
end
|
154
|
+
|
155
|
+
length = record.flowset_length - 4
|
156
|
+
|
157
|
+
# Template shouldn't be longer than the record and there should
|
158
|
+
# be at most 3 padding bytes
|
159
|
+
if template.num_bytes > length or ! (length % template.num_bytes).between?(0, 3)
|
160
|
+
$log.warn("Template length doesn't fit cleanly into flowset", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length)
|
161
|
+
next
|
162
|
+
end
|
163
|
+
|
164
|
+
array = BinData::Array.new(:type => template, :initial_length => length / template.num_bytes)
|
165
|
+
|
166
|
+
records = array.read(record.flowset_data)
|
167
|
+
records.each do |r|
|
168
|
+
time = flowset.unix_sec
|
169
|
+
event = {}
|
170
|
+
|
171
|
+
# Fewer fields in the v9 header
|
172
|
+
['version', 'flow_seq_num'].each do |f|
|
173
|
+
event[f] = flowset[f]
|
174
|
+
end
|
175
|
+
|
176
|
+
event['flowset_id'] = record.flowset_id
|
177
|
+
|
178
|
+
r.each_pair do |k,v|
|
179
|
+
case k.to_s
|
180
|
+
when /_switched$/
|
181
|
+
millis = flowset.uptime - v
|
182
|
+
seconds = flowset.unix_sec - (millis / 1000)
|
183
|
+
# v9 did away with the nanosecs field
|
184
|
+
micros = 1000000 - (millis % 1000)
|
185
|
+
event[k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
|
186
|
+
else
|
187
|
+
event[k.to_s] = v
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
yield time, event
|
192
|
+
end
|
193
|
+
else
|
194
|
+
$log.warn("Unsupported flowset id #{record.flowset_id}")
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
private
|
201
|
+
|
202
|
+
def uint_field(length, default)
|
203
|
+
# If length is 4, return :uint32, etc. and use default if length is 0
|
204
|
+
("uint" + (((length > 0) ? length : default) * 8).to_s).to_sym
|
205
|
+
end
|
206
|
+
|
207
|
+
def netflow_field_for(type, length)
|
208
|
+
if @fields.include?(type)
|
209
|
+
field = @fields[type]
|
210
|
+
if field.is_a?(Array)
|
211
|
+
|
212
|
+
if field[0].is_a?(Integer)
|
213
|
+
field[0] = uint_field(length, field[0])
|
214
|
+
end
|
215
|
+
|
216
|
+
# Small bit of fixup for skip or string field types where the length
|
217
|
+
# is dynamic
|
218
|
+
case field[0]
|
219
|
+
when :skip
|
220
|
+
field += [nil, {:length => length}]
|
221
|
+
when :string
|
222
|
+
field += [{:length => length, :trim_padding => true}]
|
223
|
+
end
|
224
|
+
|
225
|
+
[field]
|
226
|
+
else
|
227
|
+
$log.warn("Definition should be an array", :field => field)
|
228
|
+
nil
|
229
|
+
end
|
230
|
+
else
|
231
|
+
$log.warn("Unsupported field", :type => type, :length => length)
|
232
|
+
nil
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
class IP4Addr < BinData::Primitive
|
237
|
+
endian :big
|
238
|
+
uint32 :storage
|
239
|
+
|
240
|
+
def set(val)
|
241
|
+
ip = IPAddr.new(val)
|
242
|
+
if ! ip.ipv4?
|
243
|
+
raise ArgumentError, "invalid IPv4 address '#{val}'"
|
244
|
+
end
|
245
|
+
self.storage = ip.to_i
|
246
|
+
end
|
247
|
+
|
248
|
+
def get
|
249
|
+
IPAddr.new_ntoh([self.storage].pack('N')).to_s
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
class IP6Addr < BinData::Primitive
|
254
|
+
endian :big
|
255
|
+
uint128 :storage
|
256
|
+
|
257
|
+
def set(val)
|
258
|
+
ip = IPAddr.new(val)
|
259
|
+
if ! ip.ipv6?
|
260
|
+
raise ArgumentError, "invalid IPv6 address `#{val}'"
|
261
|
+
end
|
262
|
+
self.storage = ip.to_i
|
263
|
+
end
|
264
|
+
|
265
|
+
def get
|
266
|
+
IPAddr.new_ntoh((0..7).map { |i|
|
267
|
+
(self.storage >> (112 - 16 * i)) & 0xffff
|
268
|
+
}.pack('n8')).to_s
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
272
|
+
class MacAddr < BinData::Primitive
|
273
|
+
array :bytes, :type => :uint8, :initial_length => 6
|
274
|
+
|
275
|
+
def set(val)
|
276
|
+
ints = val.split(/:/).collect { |int| int.to_i(16) }
|
277
|
+
self.bytes = ints
|
278
|
+
end
|
279
|
+
|
280
|
+
def get
|
281
|
+
self.bytes.collect { |byte| byte.to_s(16) }.join(":")
|
282
|
+
end
|
283
|
+
end
|
284
|
+
|
285
|
+
class Header < BinData::Record
|
286
|
+
endian :big
|
287
|
+
uint16 :version
|
288
|
+
end
|
289
|
+
|
290
|
+
class Netflow5PDU < BinData::Record
|
291
|
+
endian :big
|
292
|
+
uint16 :version
|
293
|
+
uint16 :flow_records
|
294
|
+
uint32 :uptime
|
295
|
+
uint32 :unix_sec
|
296
|
+
uint32 :unix_nsec
|
297
|
+
uint32 :flow_seq_num
|
298
|
+
uint8 :engine_type
|
299
|
+
uint8 :engine_id
|
300
|
+
bit2 :sampling_algorithm
|
301
|
+
bit14 :sampling_interval
|
302
|
+
array :records, :initial_length => :flow_records do
|
303
|
+
ip4_addr :ipv4_src_addr
|
304
|
+
ip4_addr :ipv4_dst_addr
|
305
|
+
ip4_addr :ipv4_next_hop
|
306
|
+
uint16 :input_snmp
|
307
|
+
uint16 :output_snmp
|
308
|
+
uint32 :in_pkts
|
309
|
+
uint32 :in_bytes
|
310
|
+
uint32 :first_switched
|
311
|
+
uint32 :last_switched
|
312
|
+
uint16 :l4_src_port
|
313
|
+
uint16 :l4_dst_port
|
314
|
+
skip :length => 1
|
315
|
+
uint8 :tcp_flags # Split up the TCP flags maybe?
|
316
|
+
uint8 :protocol
|
317
|
+
uint8 :src_tos
|
318
|
+
uint16 :src_as
|
319
|
+
uint16 :dst_as
|
320
|
+
uint8 :src_mask
|
321
|
+
uint8 :dst_mask
|
322
|
+
skip :length => 2
|
323
|
+
end
|
324
|
+
end
|
325
|
+
|
326
|
+
class TemplateFlowset < BinData::Record
|
327
|
+
endian :big
|
328
|
+
array :templates, :read_until => lambda { array.num_bytes == flowset_length - 4 } do
|
329
|
+
uint16 :template_id
|
330
|
+
uint16 :field_count
|
331
|
+
array :fields, :initial_length => :field_count do
|
332
|
+
uint16 :field_type
|
333
|
+
uint16 :field_length
|
334
|
+
end
|
335
|
+
end
|
336
|
+
end
|
337
|
+
|
338
|
+
class OptionFlowset < BinData::Record
|
339
|
+
endian :big
|
340
|
+
array :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
|
341
|
+
uint16 :template_id
|
342
|
+
uint16 :scope_length
|
343
|
+
uint16 :option_length
|
344
|
+
array :scope_fields, :initial_length => lambda { scope_length / 4 } do
|
345
|
+
uint16 :field_type
|
346
|
+
uint16 :field_length
|
347
|
+
end
|
348
|
+
array :option_fields, :initial_length => lambda { option_length / 4 } do
|
349
|
+
uint16 :field_type
|
350
|
+
uint16 :field_length
|
351
|
+
end
|
352
|
+
end
|
353
|
+
skip :length => lambda { templates.length.odd? ? 2 : 0 }
|
354
|
+
end
|
355
|
+
|
356
|
+
class Netflow9PDU < BinData::Record
|
357
|
+
endian :big
|
358
|
+
uint16 :version
|
359
|
+
uint16 :flow_records
|
360
|
+
uint32 :uptime
|
361
|
+
uint32 :unix_sec
|
362
|
+
uint32 :flow_seq_num
|
363
|
+
uint32 :source_id
|
364
|
+
array :records, :read_until => :eof do
|
365
|
+
uint16 :flowset_id
|
366
|
+
uint16 :flowset_length
|
367
|
+
choice :flowset_data, :selection => :flowset_id do
|
368
|
+
template_flowset 0
|
369
|
+
option_flowset 1
|
370
|
+
string :default, :read_length => lambda { flowset_length - 4 }
|
371
|
+
end
|
372
|
+
end
|
373
|
+
end
|
374
|
+
|
375
|
+
# https://gist.github.com/joshaven/184837
|
376
|
+
class Vash < Hash
|
377
|
+
def initialize(constructor = {})
|
378
|
+
@register ||= {}
|
379
|
+
if constructor.is_a?(Hash)
|
380
|
+
super()
|
381
|
+
merge(constructor)
|
382
|
+
else
|
383
|
+
super(constructor)
|
384
|
+
end
|
385
|
+
end
|
386
|
+
|
387
|
+
alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
|
388
|
+
alias_method :regular_reader, :[] unless method_defined?(:regular_reader)
|
389
|
+
|
390
|
+
def [](key)
|
391
|
+
sterilize(key)
|
392
|
+
clear(key) if expired?(key)
|
393
|
+
regular_reader(key)
|
394
|
+
end
|
395
|
+
|
396
|
+
def []=(key, *args)
|
397
|
+
if args.length == 2
|
398
|
+
value, ttl = args[1], args[0]
|
399
|
+
elsif args.length == 1
|
400
|
+
value, ttl = args[0], 60
|
401
|
+
else
|
402
|
+
raise ArgumentError, "Wrong number of arguments, expected 2 or 3, received: #{args.length+1}\n"+
|
403
|
+
"Example Usage: volatile_hash[:key]=value OR volatile_hash[:key, ttl]=value"
|
404
|
+
end
|
405
|
+
sterilize(key)
|
406
|
+
ttl(key, ttl)
|
407
|
+
regular_writer(key, value)
|
408
|
+
end
|
409
|
+
|
410
|
+
def merge(hsh)
|
411
|
+
hsh.map {|key,value| self[sterile(key)] = hsh[key]}
|
412
|
+
self
|
413
|
+
end
|
414
|
+
|
415
|
+
def cleanup!
|
416
|
+
now = Time.now.to_i
|
417
|
+
@register.map {|k,v| clear(k) if v < now}
|
418
|
+
end
|
419
|
+
|
420
|
+
def clear(key)
|
421
|
+
sterilize(key)
|
422
|
+
@register.delete key
|
423
|
+
self.delete key
|
424
|
+
end
|
425
|
+
|
426
|
+
private
|
427
|
+
|
428
|
+
def expired?(key)
|
429
|
+
Time.now.to_i > @register[key].to_i
|
430
|
+
end
|
431
|
+
|
432
|
+
def ttl(key, secs=60)
|
433
|
+
@register[key] = Time.now.to_i + secs.to_i
|
434
|
+
end
|
435
|
+
|
436
|
+
def sterile(key)
|
437
|
+
String === key ? key.chomp('!').chomp('=') : key.to_s.chomp('!').chomp('=').to_sym
|
438
|
+
end
|
439
|
+
|
440
|
+
def sterilize(key)
|
441
|
+
key = sterile(key)
|
442
|
+
end
|
443
|
+
end
|
444
|
+
end
|
445
|
+
|
446
|
+
register_template('netflow', self)
|
447
|
+
end
|
448
|
+
end
|
metadata
ADDED
@@ -0,0 +1,94 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-netflow
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Masahiro Nakagawa
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-06-20 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.10.17
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.10.17
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bindata
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '2.1'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '2.1'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.9.2
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.9.2
|
55
|
+
description: Netflow plugin for Fluentd
|
56
|
+
email: repeatedly@gmail.com
|
57
|
+
executables: []
|
58
|
+
extensions: []
|
59
|
+
extra_rdoc_files: []
|
60
|
+
files:
|
61
|
+
- ".travis.yml"
|
62
|
+
- Gemfile
|
63
|
+
- README.md
|
64
|
+
- Rakefile
|
65
|
+
- VERSION
|
66
|
+
- fluent-plugin-netflow.gemspec
|
67
|
+
- lib/fluent/plugin/in_netflow.rb
|
68
|
+
- lib/fluent/plugin/netflow.yaml
|
69
|
+
- lib/fluent/plugin/parser_netflow.rb
|
70
|
+
homepage: https://github.com/repeatedly/fluent-plugin-netflow
|
71
|
+
licenses:
|
72
|
+
- Apache License (2.0)
|
73
|
+
metadata: {}
|
74
|
+
post_install_message:
|
75
|
+
rdoc_options: []
|
76
|
+
require_paths:
|
77
|
+
- lib
|
78
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
84
|
+
requirements:
|
85
|
+
- - ">="
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
version: '0'
|
88
|
+
requirements: []
|
89
|
+
rubyforge_project:
|
90
|
+
rubygems_version: 2.2.2
|
91
|
+
signing_key:
|
92
|
+
specification_version: 4
|
93
|
+
summary: Netflow plugin for Fluentd
|
94
|
+
test_files: []
|